Merge branch 'rickard/gc-bump-reds/OTP-13097'

* rickard/gc-bump-reds/OTP-13097:
  Bump reductions on GC

10 files changed, 159 insertions, 108 deletions

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 22b4e26c77..c3ebf71a01 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -935,7 +935,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
 
 	if (need_gc & ERTS_ORDINARY_GC__) {
 	    FLAGS(rp) |= F_NEED_FULLSWEEP;
-	    *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+	    *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity);
 	    done_gc |= ERTS_ORDINARY_GC__;
 	}
 	if (need_gc & ERTS_LITERAL_GC__) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 162713622b..208a16dfd0 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -237,6 +237,14 @@ void** beam_ops;
     HEAP_TOP(c_p) = HTOP;  \
     c_p->stop = E
 
+#define HEAVY_SWAPIN       \
+    SWAPIN;		   \
+    FCALLS = c_p->fcalls
+
+#define HEAVY_SWAPOUT      \
+    SWAPOUT;		   \
+    c_p->fcalls = FCALLS
+
 /*
  * Use LIGHT_SWAPOUT when the called function
  * will call HeapOnlyAlloc() (and never HAlloc()).
@@ -297,7 +305,7 @@ void** beam_ops;
      if (E - HTOP < (needed + (HeapNeed))) { \
            SWAPOUT; \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
-           FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
+           FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \
            ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
            SWAPIN; \
@@ -349,7 +357,7 @@ void** beam_ops;
     if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
        SWAPOUT;                                                 		\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
-       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  		\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));  		\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
        SWAPIN;                                                  		\
@@ -370,7 +378,7 @@ void** beam_ops;
     if (E - HTOP < need) {                                      \
        SWAPOUT;                                                 \
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
-       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  \
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
        SWAPIN;                                                  \
@@ -391,7 +399,7 @@ void** beam_ops;
        SWAPOUT;								\
        reg[Live] = Extra;						\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
-       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1);	\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1);	\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
        Extra = reg[Live];						\
@@ -415,9 +423,9 @@ void** beam_ops;
 
 #define MakeFun(FunP, NumFree)					\
   do {								\
-     SWAPOUT;							\
+     HEAVY_SWAPOUT;						\
      r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree);	\
-     SWAPIN;							\
+     HEAVY_SWAPIN;						\
   } while (0)
 
 #define PutTuple(Dst, Arity)			\
@@ -1402,11 +1410,11 @@ void process_main(void)
 	    }
 
 	    live = Arg(2);
-	    SWAPOUT;
+	    HEAVY_SWAPOUT;
 	    reg[live] = increment_reg_val;
 	    reg[live+1] = make_small(increment_val);
 	    result = erts_gc_mixed_plus(c_p, reg, live);
-	    SWAPIN;
+	    HEAVY_SWAPIN;
 	    ERTS_HOLE_CHECK(c_p);
 	    if (is_value(result)) {
 		StoreBifResult(3, result);
@@ -1420,11 +1428,11 @@ void process_main(void)
      Eterm result;				\
      Uint live = Arg(1);			\
 						\
-     SWAPOUT;					\
+     HEAVY_SWAPOUT;				\
      reg[live] = Op1;				\
      reg[live+1] = Op2;				\
      result = erts_gc_##name(c_p, reg, live);	\
-     SWAPIN;					\
+     HEAVY_SWAPIN;				\
      ERTS_HOLE_CHECK(c_p);			\
      if (is_value(result)) {			\
 	 StoreBifResult(4, result);		\
@@ -1744,7 +1752,7 @@ void process_main(void)
 	     if (E - HTOP < 3) {
 		 SWAPOUT;
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
-		 FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
+		 FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1);
 		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 SWAPIN;
@@ -2382,9 +2390,9 @@ void process_main(void)
  OpCase(new_map_dII): {
      Eterm res;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      res = new_map(c_p, reg, I-1);
-     SWAPIN;
+     HEAVY_SWAPIN;
      StoreResult(res, Arg(0));
      Next(3+Arg(2));
  }
@@ -2472,9 +2480,9 @@ do {						\
      Eterm map;
 
      GetArg1(1, map);
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      res = update_map_assoc(c_p, reg, map, I);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (is_value(res)) {
 	 StoreResult(res, Arg(2));
 	 Next(5+Arg(4));
@@ -2494,9 +2502,9 @@ do {						\
      Eterm map;
 
      GetArg1(1, map);
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      res = update_map_exact(c_p, reg, map, I);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (is_value(res)) {
 	 StoreResult(res, Arg(2));
 	 Next(5+Arg(4));
@@ -3072,10 +3080,10 @@ do {						\
 	 bnot_val = make_small(~signed_val(bnot_val));
      } else {
 	 Uint live = Arg(2);
-	 SWAPOUT;
+	 HEAVY_SWAPOUT;
 	 reg[live] = bnot_val;
 	 bnot_val = erts_gc_bnot(c_p, reg, live);
-	 SWAPIN;
+	 HEAVY_SWAPIN;
 	 ERTS_HOLE_CHECK(c_p);
 	 if (is_nil(bnot_val)) {
 	     goto lb_Cl_error;
@@ -3090,9 +3098,9 @@ do {						\
 
  OpCase(i_apply): {
      BeamInstr *next;
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+1);
 	 SET_I(next);
@@ -3104,9 +3112,9 @@ do {						\
 
  OpCase(i_apply_last_P): {
      BeamInstr *next;
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
@@ -3119,9 +3127,9 @@ do {						\
 
  OpCase(i_apply_only): {
      BeamInstr *next;
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply(c_p, r(0), x(1), x(2), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_I(next);
 	 Dispatch();
@@ -3133,9 +3141,9 @@ do {						\
  OpCase(apply_I): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = fixed_apply(c_p, reg, Arg(0));
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+2);
 	 SET_I(next);
@@ -3148,9 +3156,9 @@ do {						\
  OpCase(apply_last_IP): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = fixed_apply(c_p, reg, Arg(0));
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(1));
@@ -3164,9 +3172,9 @@ do {						\
  OpCase(i_apply_fun): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+1);
 	 SET_I(next);
@@ -3178,9 +3186,9 @@ do {						\
  OpCase(i_apply_fun_last_P): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
 	 E = ADD_BYTE_OFFSET(E, Arg(0));
@@ -3193,9 +3201,9 @@ do {						\
  OpCase(i_apply_fun_only): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = apply_fun(c_p, r(0), x(1), reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_I(next);
 	 Dispatchfun();
@@ -3206,9 +3214,9 @@ do {						\
  OpCase(i_call_fun_I): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+2);
 	 SET_I(next);
@@ -3220,9 +3228,9 @@ do {						\
  OpCase(i_call_fun_last_IP): {
      BeamInstr *next;
 
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (next != NULL) {
 	SET_CP(c_p, (BeamInstr *) E[0]);
 	E = ADD_BYTE_OFFSET(E, Arg(1));
@@ -3421,9 +3429,9 @@ do {						\
      * code[3]: &&call_error_handler
      * code[4]: Not used
      */
-    SWAPOUT;
+    HEAVY_SWAPOUT;
     I = call_error_handler(c_p, I-3, reg, am_undefined_function);
-    SWAPIN;
+    HEAVY_SWAPIN;
     if (I) {
 	Goto(*I);
     }
@@ -3977,10 +3985,10 @@ do {						\
      Eterm Size;
 
      GetArg1(4, Size);
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      reg[live] = x(SCRATCH_X_REG);
      res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3));
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (is_non_value(res)) {
 	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
 	 goto lb_Cl_error;
@@ -4005,9 +4013,9 @@ do {						\
  }
 
  OpCase(bs_init_writable): {
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      r(0) = erts_bs_init_writable(c_p, r(0));
-     SWAPIN;
+     HEAVY_SWAPIN;
      Next(0);
  }
 
@@ -4835,7 +4843,7 @@ do {						\
 	     BeamInstr *next;
 
 	     next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
-	     SWAPIN;
+	     HEAVY_SWAPIN;
 	     if (next != NULL) {
 		 SET_I(next);
 		 Dispatchfun();
@@ -4884,20 +4892,22 @@ do {						\
  }
 
  OpCase(i_hibernate): {
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+	 FCALLS = c_p->fcalls;
 	 c_p->flags &= ~F_HIBERNATE_SCHED;
 	 goto do_schedule;
      } else {
+	 HEAVY_SWAPIN;
 	 I = handle_error(c_p, I, reg, hibernate_3);
 	 goto post_error_handling;
      }
  }
 
  OpCase(i_debug_breakpoint): {
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      I = call_error_handler(c_p, I-3, reg, am_breakpoint);
-     SWAPIN;
+     HEAVY_SWAPIN;
      if (I) {
 	 Goto(*I);
      }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d415603930..f0340540cb 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3839,11 +3839,9 @@ BIF_RETTYPE now_0(BIF_ALIST_0)
 
 BIF_RETTYPE garbage_collect_0(BIF_ALIST_0)
 {
-    int reds;
-
     FLAGS(BIF_P) |= F_NEED_FULLSWEEP;
-    reds = erts_garbage_collect(BIF_P, 0, NULL, 0);
-    BIF_RET2(am_true, reds);
+    erts_garbage_collect(BIF_P, 0, NULL, 0);
+    BIF_RET(am_true);
 }
 
 /**********************************************************************/
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index c8f40ee162..6cb37752bc 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -135,7 +135,7 @@ static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm
 					 char* src, Uint src_size);
 static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
 				      Eterm* heap, Eterm* htop, Eterm* objv, int nobj);
-static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
+static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
 static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
 static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
 static void sweep_off_heap(Process *p, int fullsweep);
@@ -172,6 +172,20 @@ typedef struct {
     erts_smp_atomic32_t refc;
 } ErtsGCInfoReq;
 
+static ERTS_INLINE int
+gc_cost(Uint gc_moved_live_words, Uint resize_moved_words)
+{
+    Sint reds;
+
+    reds = gc_moved_live_words/10;
+    reds += resize_moved_words/100;
+    if (reds < 1)
+	return 1;
+    if (reds > INT_MAX)
+	return INT_MAX;
+    return (int) reds;
+}
+
 ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq,
                                  ErtsGCInfoReq,
                                  5,
@@ -413,6 +427,7 @@ static ERTS_INLINE void reset_active_writer(Process *p)
 }
 
 #define ERTS_DELAY_GC_EXTRA_FREE 40
+#define ERTS_ABANDON_HEAP_COST 10
 
 static int
 delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
@@ -421,6 +436,7 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
     Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop;
     Eterm *stop, *hend;
     Uint hsz, ssz;
+    int reds_left;
 
     ERTS_HOLE_CHECK(p);
 
@@ -482,7 +498,12 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
 
     /* Make sure that we do a proper GC as soon as possible... */
     p->flags |= F_FORCE_GC;
-    return CONTEXT_REDS;
+    reds_left = ERTS_BIF_REDS_LEFT(p);
+    if (reds_left > ERTS_ABANDON_HEAP_COST) {
+	int vreds = reds_left - ERTS_ABANDON_HEAP_COST;
+	ERTS_VBUMP_REDS(p, vreds);
+    }
+    return ERTS_ABANDON_HEAP_COST;
 }
 
 static ERTS_FORCE_INLINE Uint
@@ -536,7 +557,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 		int need, Eterm* objv, int nobj)
 {
     Uint reclaimed_now = 0;
-    int done = 0;
+    int reds;
     ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
     ErtsSchedulerData *esdp;
 #ifdef USE_VM_PROBES
@@ -562,9 +583,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     ERTS_CHK_OFFHEAP(p);
 
     ErtsGcQuickSanityCheck(p);
-    if (GEN_GCS(p) >= MAX_GEN_GCS(p)) {
-        FLAGS(p) |= F_NEED_FULLSWEEP;
-    }
+
 #ifdef USE_VM_PROBES
     *pidbuf = '\0';
     if (DTRACE_ENABLED(gc_major_start)
@@ -577,17 +596,21 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     /*
      * Test which type of GC to do.
      */
-    while (!done) {
-	if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) {
-	    DTRACE2(gc_major_start, pidbuf, need);
-	    done = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
-	    DTRACE2(gc_major_end, pidbuf, reclaimed_now);
-	} else {
-	    DTRACE2(gc_minor_start, pidbuf, need);
-	    done = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
-	    DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
-	}
+
+    if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) {
+	DTRACE2(gc_minor_start, pidbuf, need);
+	reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+	DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
+	if (reds < 0)
+	    goto do_major_collection;
+    }
+    else {
+    do_major_collection:
+	DTRACE2(gc_major_start, pidbuf, need);
+	reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+	DTRACE2(gc_major_end, pidbuf, reclaimed_now);
     }
+
     reset_active_writer(p);
 
     /*
@@ -648,23 +671,22 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     p->last_old_htop = p->old_htop;
 #endif
 
-    /* FIXME: This function should really return an Sint, i.e., a possibly
-       64 bit wide signed integer, but that requires updating all the code
-       that calls it. For now, we just return INT_MAX if the result is too
-       large for an int. */
-    {
-      Sint result = (HEAP_TOP(p) - HEAP_START(p)) / 10;
-      if (result >= INT_MAX) return INT_MAX;
-      else return (int) result;
-    }
+    return reds;
 }
 
 int
-erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
+erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj)
 {
     return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
 }
 
+void
+erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
+{
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+    BUMP_REDS(p, reds);
+}
+
 /*
  * Place all living data on a the new heap; deallocate any old heap.
  * Meant to be used by hibernate/3.
@@ -679,6 +701,7 @@ erts_garbage_collect_hibernate(Process* p)
     char* area;
     Uint area_size;
     Sint offs;
+    int reds;
 
     if (p->flags & F_DISABLE_GC)
 	ERTS_INTERNAL_ERROR("GC disabled");
@@ -774,6 +797,9 @@ erts_garbage_collect_hibernate(Process* p)
     ErtsGcQuickSanityCheck(p);
 
     erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+
+    reds = gc_cost(actual_size, actual_size);
+    BUMP_REDS(p, reds);
 }
 
 
@@ -988,16 +1014,21 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
     if (OLD_HEAP(p) &&
 	((mature_size <= OLD_HEND(p) - OLD_HTOP(p)) &&
 	 ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
-	Uint stack_size, size_after, need_after, new_sz;
+	Eterm *prev_old_htop;
+	Uint stack_size, size_after, adjust_size, need_after, new_sz, new_mature;
 
 	stack_size = p->hend - p->stop;
 	new_sz = stack_size + size_before;
         new_sz = next_heap_size(p, new_sz, 0);
 
+	prev_old_htop = p->old_htop;
         do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm),
 		 new_sz, objv, nobj);
 
-        size_after = HEAP_TOP(p) - HEAP_START(p);
+	new_mature = p->old_htop - prev_old_htop;
+
+	size_after = new_mature;
+        size_after += HEAP_TOP(p) - HEAP_START(p);
         *recl += (size_before - size_after);
 
 	ErtsGcQuickSanityCheck(p);
@@ -1016,6 +1047,8 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
          * the heap size is substantial, we don't want to shrink.
          */
 
+	adjust_size = 0;
+
         if ((HEAP_SIZE(p) > 3000) && (4 * need_after < HEAP_SIZE(p)) &&
             ((HEAP_SIZE(p) > 8000) ||
              (HEAP_SIZE(p) > (OLD_HEND(p) - OLD_HEAP(p))))) {
@@ -1037,31 +1070,33 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
 					       : next_heap_size(p, wanted, 0);
             if (wanted < HEAP_SIZE(p)) {
                 shrink_new_heap(p, wanted, objv, nobj);
+		adjust_size = p->htop - p->heap;
             }
 
-            ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
-            ASSERT(MBUF(p) == NULL);
-	    return 1;		/* We are done. */
+	    goto done;
         }
 
         if (HEAP_SIZE(p) >= need_after) {
 	    /*
 	     * The heap size turned out to be just right. We are done.
 	     */
-            ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
-            ASSERT(MBUF(p) == NULL);
-            return 1;
+	    goto done;
 	}
 
         grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj);
-	return 1;
+	adjust_size = p->htop - p->heap;
+
+    done:
+	ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
+	ASSERT(MBUF(p) == NULL);
+
+	return gc_cost(size_after, adjust_size);
     }
 
     /*
      * Not enough room for a minor collection. Must force a major collection.
      */
-    FLAGS(p) |= F_NEED_FULLSWEEP;
-    return 0;
+    return -1;
 }
 
 /*
@@ -1340,12 +1375,13 @@ static int
 major_collection(Process* p, ErlHeapFragment *live_hf_end,
 		 int need, Eterm* objv, int nobj, Uint *recl)
 {
-    Uint size_before, stack_size;
+    Uint size_before, size_after, stack_size;
     Eterm* n_heap;
     Eterm* n_htop;
     char* oh = (char *) OLD_HEAP(p);
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
     Uint new_sz, stk_sz;
+    int adjusted;
 
     /*
      * Do a fullsweep GC. First figure out the size of the heap
@@ -1412,9 +1448,10 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 
     ErtsGcQuickSanityCheck(p);
 
-    *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
+    size_after = HEAP_TOP(p) - HEAP_START(p);
+    *recl += size_before - size_after;
 
-    adjust_after_fullsweep(p, need, objv, nobj);
+    adjusted = adjust_after_fullsweep(p, need, objv, nobj);
 
 #ifdef HARDDEBUG
     disallow_heap_frag_ref_in_heap(p);
@@ -1422,7 +1459,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     remove_message_buffers(p);
 
     ErtsGcQuickSanityCheck(p);
-    return 1;			/* We are done. */
+
+    return gc_cost(size_after, adjusted ? size_after : 0);
 }
 
 static Eterm *
@@ -1523,9 +1561,10 @@ full_sweep_heaps(Process *p,
     return n_htop;
 }
 
-static void
+static int
 adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
 {
+    int adjusted = 0;
     Uint wanted, sz, need_after;
     Uint stack_size = STACK_SZ_ON_HEAP(p);
     
@@ -1538,6 +1577,7 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
         /* Too small - grow to match requested need */
         sz = next_heap_size(p, need_after, 0);
         grow_new_heap(p, sz, objv, nobj);
+	adjusted = 1;
     } else if (3 * HEAP_SIZE(p) < 4 * need_after){
         /* Need more than 75% of current, postpone to next GC.*/
         FLAGS(p) |= F_HEAP_GROW;
@@ -1554,8 +1594,10 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
 
         if (sz < HEAP_SIZE(p)) {
             shrink_new_heap(p, sz, objv, nobj);
+	    adjusted = 1;
         }
     }
+    return adjusted;
 }
 
 /*
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index a496c5f008..d603866cbf 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -134,7 +134,8 @@ typedef struct {
 
 void erts_gc_info(ErtsGCInfo *gcip);
 void erts_init_gc(void);
-int erts_garbage_collect(struct process*, int, Eterm*, int);
+int erts_garbage_collect_nobump(struct process*, int, Eterm*, int);
+void erts_garbage_collect(struct process*, int, Eterm*, int);
 void erts_garbage_collect_hibernate(struct process* p);
 Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end,
 				 Eterm result, Eterm* regs, Uint arity);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6e83456e4c..9acce8acb6 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9779,7 +9779,7 @@ Process *schedule(Process *p, int calls)
 
 	if (ERTS_IS_GC_DESIRED(p)) {
 	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & F_DISABLE_GC)) {
-		reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity);
+		reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity);
 		if (reds <= 0) {
 		    p->fcalls = reds;
 		    goto sched_out_proc;
@@ -10062,10 +10062,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    else {
 		if (!garbage_collected) {
 		    FLAGS(c_p) |= F_NEED_FULLSWEEP;
-		    reds += erts_garbage_collect(c_p,
-						 0,
-						 c_p->arg_reg,
-						 c_p->arity);
+		    reds += erts_garbage_collect_nobump(c_p,
+							0,
+							c_p->arg_reg,
+							c_p->arity);
 		    garbage_collected = 1;
 		}
 		st_res = am_true;
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 8606371bdf..f82cad745a 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -583,7 +583,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
 	root[0] = id;
 	root[1] = value;
 	root[2] = old;
-	BUMP_REDS(p, erts_garbage_collect(p, needed, root, 3));
+	erts_garbage_collect(p, needed, root, 3);
 	id = root[0];
 	value = root[1];
 	old = root[2];
@@ -715,7 +715,7 @@ static void shrink(Process *p, Eterm* ret)
 		    needed = 2*erts_list_length(hi);
 		}
 		if (HeapWordsLeft(p) < needed) {
-		    BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1));
+		    erts_garbage_collect(p, needed, ret, 1);
 		    hi = pd->data[(pd->splitPosition + pd->homeSize)];
 		    lo = pd->data[pd->splitPosition];
 		}
@@ -811,7 +811,7 @@ static void grow(Process *p)
 	}
     }
     if (HeapWordsLeft(p) < needed) {
-	BUMP_REDS(p, erts_garbage_collect(p, needed, 0, 0));
+	erts_garbage_collect(p, needed, 0, 0);
     }
 #ifdef DEBUG
     hp_limit = p->htop + needed;
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 968452a641..976180cc30 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -196,7 +196,7 @@ hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
 	ASSERT(!(p->flags & F_DISABLE_GC));
 	if ((p->stop - 2) < p->htop) {
 	    DPRINTF("calling gc to increase BEAM stack size");
-	    p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+	    erts_garbage_collect(p, 2, reg, arity);
 	    ASSERT(!((p->stop - 2) < p->htop));
 	}
 	p->stop -= 2;
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index bc863a4f36..620cc6356b 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -95,7 +95,7 @@ ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process *p, Eterm reg[], unsig
 
     /* ensure that at least 2 words are available on the BEAM stack */
     if ((p->stop - 2) < p->htop) {
-	p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+	erts_garbage_collect(p, 2, reg, arity);
 	ASSERT(!((p->stop - 2) < p->htop));
     }
     p->stop -= 2;
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index 089c0cc662..ad8fb685e5 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -80,7 +80,7 @@ Eterm hipe_show_nstack_1(BIF_ALIST_1)
 void hipe_gc(Process *p, Eterm need)
 {
     hipe_set_narity(p, 1);
-    p->fcalls -= erts_garbage_collect(p, unsigned_val(need), NULL, 0);
+    erts_garbage_collect(p, unsigned_val(need), NULL, 0);
     hipe_set_narity(p, 0);
 }