Merge branch 'rickard/reds-fix/master/OTP-13512'

* rickard/reds-fix/master/OTP-13512:
  Ensure correct reduction counting

13 files changed, 308 insertions, 108 deletions

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 87508dcf5f..801218774b 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -38,7 +38,7 @@
 #include "erl_thr_progress.h"
 
 static void set_default_trace_pattern(Eterm module);
-static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp);
+static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls);
 static void delete_code(Module* modp);
 static void decrement_refc(BeamCodeHeader*);
 static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size);
@@ -467,7 +467,7 @@ check_old_code_1(BIF_ALIST_1)
 }
 
 Eterm
-erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp)
+erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls)
 {
     Module* modp;
     Eterm res;
@@ -483,7 +483,7 @@ erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp)
 	return am_false;
     erts_rlock_old_code(code_ix);
     res = (!modp->old.code_hdr ? am_false :
-           check_process_code(c_p, modp, flags, redsp));
+           check_process_code(c_p, modp, flags, redsp, fcalls));
     erts_runlock_old_code(code_ix);
 
     return res;
@@ -506,7 +506,7 @@ BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2)
         goto badarg;
     }
 
-    res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds);
+    res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds, BIF_P->fcalls);
 
     ASSERT(is_value(res));
 
@@ -753,7 +753,7 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size)
 
 
 static Eterm
-check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
+check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls)
 {
     BeamInstr* start;
     char* literals;
@@ -955,7 +955,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp)
 
 	if (need_gc & ERTS_ORDINARY_GC__) {
 	    FLAGS(rp) |= F_NEED_FULLSWEEP;
-	    *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity);
+	    *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls);
 	    done_gc |= ERTS_ORDINARY_GC__;
 	}
 	if (need_gc & ERTS_LITERAL_GC__) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 72526bca5e..59112e1e43 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -308,7 +308,8 @@ void** beam_ops;
      if (E - HTOP < (needed + (HeapNeed))) { \
            SWAPOUT; \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
-           FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \
+           FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), \
+						 reg, (M), FCALLS);	\
            ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
            PROCESS_MAIN_CHK_LOCKS(c_p); \
            SWAPIN; \
@@ -360,7 +361,7 @@ void** beam_ops;
     if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
        SWAPOUT;                                                 		\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));  		\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS);	\
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
        SWAPIN;                                                  		\
@@ -381,7 +382,7 @@ void** beam_ops;
     if (E - HTOP < need) {                                      \
        SWAPOUT;                                                 \
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live), FCALLS); \
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
        PROCESS_MAIN_CHK_LOCKS(c_p);                             \
        SWAPIN;                                                  \
@@ -402,7 +403,7 @@ void** beam_ops;
        SWAPOUT;								\
        reg[Live] = Extra;						\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
-       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1);	\
+       FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1, FCALLS); \
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
        PROCESS_MAIN_CHK_LOCKS(c_p);					\
        Extra = reg[Live];						\
@@ -1196,6 +1197,25 @@ init_emulator(void)
 #define DTRACE_NIF_RETURN(p, m, f, a)        do {} while (0)
 #endif /* USE_VM_PROBES */
 
+#ifdef DEBUG
+#define ERTS_DBG_CHK_REDS(P, FC)					\
+    do {								\
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) {			\
+	    ASSERT(FC <= 0);						\
+	    ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds		\
+		   <= 0 - (FC));					\
+	}								\
+	else {								\
+	    ASSERT(FC <= CONTEXT_REDS);					\
+	    ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds		\
+		   <= CONTEXT_REDS - (FC));				\
+	}								\
+} while (0)
+#else
+#define ERTS_DBG_CHK_REDS(P, FC)
+#endif
+
+
 /*
  * process_main() is called twice:
  * The first call performs some initialisation, including exporting
@@ -1290,7 +1310,12 @@ void process_main(void)
     goto do_schedule1;
 
  do_schedule:
-    reds_used = REDS_IN(c_p) - FCALLS;
+    ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	reds_used = REDS_IN(c_p) - FCALLS;
+    else
+	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+    ASSERT(reds_used >= 0);
  do_schedule1:
 
     if (start_time != 0) {
@@ -1310,6 +1335,7 @@ void process_main(void)
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     c_p = schedule(c_p, reds_used);
+    ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     start_time = 0;
 #ifdef DEBUG
@@ -1348,16 +1374,21 @@ void process_main(void)
 
 	SET_I(c_p->i);
 
-	reds = c_p->fcalls;
-	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
-	    && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
-	    neg_o_reds = -reds;
-	    FCALLS = REDS_IN(c_p) = 0;
+	REDS_IN(c_p) = reds = c_p->fcalls;
+#ifdef DEBUG
+	c_p->debug_reds_in = reds;
+#endif
+
+	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
+	    neg_o_reds = -CONTEXT_REDS;
+	    FCALLS = neg_o_reds + reds;
 	} else {
 	    neg_o_reds = 0;
-	    FCALLS = REDS_IN(c_p) = reds;
+	    FCALLS = reds;
 	}
 
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
 	next = (BeamInstr *) *I;
 	SWAPIN;
 	ASSERT(VALID_INSTR(next));
@@ -1770,7 +1801,7 @@ void process_main(void)
 	     if (E - HTOP < 3) {
 		 SWAPOUT;
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
-		 FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1);
+		 FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1, FCALLS);
 		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 		 PROCESS_MAIN_CHK_LOCKS(c_p);
 		 SWAPIN;
@@ -1865,6 +1896,7 @@ void process_main(void)
      c_p->flags |= F_DELAY_GC;
 
  loop_rec__:
+
      PROCESS_MAIN_CHK_LOCKS(c_p);
 
      msgp = PEEK_MESSAGE(c_p);
@@ -2007,6 +2039,8 @@ void process_main(void)
 	 ERTS_VBUMP_LEAVE_REDS_INTERNAL(c_p, 5, FCALLS);
      }
 
+     ERTS_DBG_CHK_REDS(c_p, FCALLS);
+
      ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
      PROCESS_MAIN_CHK_LOCKS(c_p);
      NextPF(0, next);
@@ -2535,6 +2569,7 @@ do {						\
 
 	GetArg1(2, tmp_reg[0]);
 	bf = (BifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2544,6 +2579,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(3, result);
 	}
@@ -2564,6 +2600,7 @@ do {						\
 
 	GetArg1(1, tmp_reg[0]);
 	bf = (BifFunction) Arg(0);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2573,6 +2610,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(2, result);
 	}
@@ -2591,6 +2629,7 @@ do {						\
 
 	GetArg1(2, x(live));
 	bf = (GcBifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2602,6 +2641,7 @@ do {						\
 	SWAPIN;
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(4, result);
 	}
@@ -2630,6 +2670,7 @@ do {						\
 	 */
 	live++;
 	bf = (GcBifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2641,6 +2682,7 @@ do {						\
 	SWAPIN;
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(5, result);
 	}
@@ -2671,6 +2713,7 @@ do {						\
 	 */
 	live += 2;
 	bf = (GcBifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	SWAPOUT;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -2682,6 +2725,7 @@ do {						\
 	SWAPIN;
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(5, result);
 	}
@@ -2708,6 +2752,7 @@ do {						\
 
 	GetArg2(2, tmp_reg[0], tmp_reg[1]);
 	bf = (BifFunction) Arg(1);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
@@ -2717,6 +2762,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ERTS_HOLE_CHECK(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	if (is_value(result)) {
 	    StoreBifResult(4, result);
 	}
@@ -2775,6 +2821,7 @@ do {						\
 	bf = GET_BIF_ADDRESS(Arg(0));
 
 	PRE_BIF_SWAPOUT(c_p);
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS - 1;
 	if (FCALLS <= 0) {
 	   save_calls(c_p, (Export *) Arg(0));
@@ -2796,6 +2843,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	HTOP = HEAP_TOP(c_p);
 	FCALLS = c_p->fcalls;
+	ERTS_DBG_CHK_REDS(c_p, FCALLS);
         /* We have to update the cache if we are enabled in order
            to make sure no book keeping is done after we disabled
            msacc. We don't always do this as it is quite expensive. */
@@ -3324,8 +3372,13 @@ do {						\
       * (beacuse the code for the Dispatch() macro becomes shorter that way).
       */
 
-     reds_used = REDS_IN(c_p) - FCALLS + 1;
-     
+     ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
+    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	reds_used = REDS_IN(c_p) - FCALLS;
+    else
+	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
+    ASSERT(reds_used >= 0);
+
      /*
       * Save the argument registers and everything else.
       */
@@ -3514,6 +3567,7 @@ do {						\
 	    DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 
 	    SWAPOUT;
+	    ERTS_DBG_CHK_REDS(c_p, FCALLS - 1);
 	    c_p->fcalls = FCALLS - 1;
 	    vbf = (BifFunction) Arg(0);
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -3549,6 +3603,7 @@ do {						\
 	    }
 	    SWAPIN;  /* There might have been a garbage collection. */
 	    FCALLS = c_p->fcalls;
+	    ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	    if (is_value(nif_bif_result)) {
 		r(0) = nif_bif_result;
 		CHECK_TERM(r(0));
@@ -4611,9 +4666,9 @@ do {						\
  OpCase(i_generic_breakpoint): {
      BeamInstr real_I;
      ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
-     SWAPOUT;
+     HEAVY_SWAPOUT;
      real_I = erts_generic_breakpoint(c_p, I, reg);
-     SWAPIN;
+     HEAVY_SWAPIN;
      ASSERT(VALID_INSTR(real_I));
      Goto(real_I);
  }
@@ -4790,6 +4845,7 @@ do {						\
  {
 #define HIPE_MODE_SWITCH(Cmd)			\
      SWAPOUT;					\
+     ERTS_DBG_CHK_REDS(c_p, FCALLS);		\
      c_p->fcalls = FCALLS;			\
      c_p->def_arg_reg[4] = -neg_o_reds;		\
      c_p = hipe_mode_switch(c_p, Cmd, reg);     \
@@ -4828,6 +4884,9 @@ do {						\
 #undef HIPE_MODE_SWITCH
 
  L_post_hipe_mode_switch:
+#ifdef DEBUG
+     pid = c_p->common.id; /* may have switched process... */
+#endif
      reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
      freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
      ERL_BITS_RELOAD_STATEP(c_p);
@@ -4835,6 +4894,7 @@ do {						\
      neg_o_reds = -c_p->def_arg_reg[4];
      FCALLS = c_p->fcalls;
      SWAPIN;
+     ERTS_DBG_CHK_REDS(c_p, FCALLS);
      switch( c_p->def_arg_reg[3] ) {
        case HIPE_MODE_SWITCH_RES_RETURN:
 	 ASSERT(is_value(reg[0]));
@@ -6839,7 +6899,7 @@ erts_current_reductions(Process *current, Process *p)
     if (current != p) {
 	return 0;
     } else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) {
-	return -current->fcalls;
+	return current->fcalls + CONTEXT_REDS;
     } else {
 	return REDS_IN(current) - current->fcalls;
     }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index c38d031ab8..9d6dfefa3c 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1569,7 +1569,32 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P,
 	   scb->n = 0;
        }
 
-       scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb);
+#ifdef HIPE
+       if (rp->flags & F_HIPE_MODE) {
+	   ASSERT(!ERTS_PROC_GET_SAVED_CALLS_BUF(rp));
+	   scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(rp, scb);
+       }
+       else
+#endif
+       {
+#ifdef HIPE
+	   ASSERT(!ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(rp));
+#endif
+	   scb = ERTS_PROC_SET_SAVED_CALLS_BUF(rp, scb);
+	   if (rp == BIF_P && ((scb && i == 0) || (!scb && i != 0))) {
+	       /* Adjust fcalls to match save calls setting... */
+	       if (i == 0)
+		   BIF_P->fcalls += CONTEXT_REDS; /* disabled it */
+	       else
+		   BIF_P->fcalls -= CONTEXT_REDS; /* enabled it */
+
+	       /*
+		* Make sure we reschedule immediately so the
+		* change take effect at once.
+		*/
+	       ERTS_VBUMP_ALL_REDS(BIF_P);
+	   }
+       }
 
        if (!scb)
 	   old_value = make_small(0);
@@ -1578,12 +1603,7 @@ static BIF_RETTYPE process_flag_aux(Process *BIF_P,
 	   erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb);
        }
 
-       /* Make sure the process in question is rescheduled
-	  immediately, if it's us, so the call saving takes effect. */
-       if (rp == BIF_P)
-	   BIF_RET2(old_value, CONTEXT_REDS);
-       else
-	   BIF_RET(old_value);
+       BIF_RET(old_value);
    }
 
  error:
@@ -1777,10 +1797,18 @@ BIF_RETTYPE process_flag_3(BIF_ALIST_3)
    Process *rp;
    Eterm res;
 
-   if ((rp = erts_pid2proc(BIF_P, ERTS_PROC_LOCK_MAIN,
-			   BIF_ARG_1, ERTS_PROC_LOCK_MAIN)) == NULL) {
+#ifdef ERTS_SMP
+   rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN,
+				  BIF_ARG_1, ERTS_PROC_LOCK_MAIN);
+   if (rp == ERTS_PROC_LOCK_BUSY)
+       ERTS_BIF_YIELD3(bif_export[BIF_process_flag_3], BIF_P,
+		       BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
+#else
+   rp = erts_proc_lookup(BIF_ARG_1);
+#endif
+
+   if (!rp)
        BIF_ERROR(BIF_P, BADARG);
-   }
 
    res = process_flag_aux(BIF_P, rp, BIF_ARG_2, BIF_ARG_3);
 
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 546e3830b9..5d751dd67d 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -52,6 +52,7 @@ extern Export *erts_convert_time_unit_trap;
 	(p)->fcalls = 0; 			\
     else 					\
 	(p)->fcalls = -CONTEXT_REDS;		\
+    ASSERT(ERTS_BIF_REDS_LEFT((p)) == 0);	\
 } while(0)
 
 #define ERTS_VBUMP_ALL_REDS_INTERNAL(p, fcalls)				\
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index cf54f1e384..e21fba6bf1 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -116,7 +116,7 @@ static Eterm *full_sweep_heaps(Process *p,
 			       char *oh, Uint oh_size,
 			       Eterm *objv, int nobj);
 static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
-			   int need, Eterm* objv, int nobj);
+			   int need, Eterm* objv, int nobj, int fcalls);
 static int major_collection(Process* p, ErlHeapFragment *live_hf_end,
 			    int need, Eterm* objv, int nobj, Uint *recl);
 static int minor_collection(Process* p, ErlHeapFragment *live_hf_end,
@@ -392,15 +392,15 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 		regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array;
 	    }
 	  #endif
-	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls);
 	} else {
-	    cost = garbage_collect(p, live_hf_end, 0, regs, arity);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls);
 	}
     } else {
 	Eterm val[1];
 
 	val[0] = result;
-	cost = garbage_collect(p, live_hf_end, 0, val, 1);
+	cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls);
 	result = val[0];
     }
     BUMP_REDS(p, cost);
@@ -431,7 +431,7 @@ static ERTS_INLINE void reset_active_writer(Process *p)
 #define ERTS_ABANDON_HEAP_COST 10
 
 static int
-delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
+delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int fcalls)
 {
     ErlHeapFragment *hfrag;
     Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop;
@@ -506,12 +506,16 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
 
     /* Make sure that we do a proper GC as soon as possible... */
     p->flags |= F_FORCE_GC;
-    reds_left = ERTS_BIF_REDS_LEFT(p);
+    reds_left = ERTS_REDS_LEFT(p, fcalls);
+    ASSERT(CONTEXT_REDS - reds_left >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+
     if (reds_left > ERTS_ABANDON_HEAP_COST) {
 	int vreds = reds_left - ERTS_ABANDON_HEAP_COST;
-	ERTS_VBUMP_REDS(p, vreds);
+	ERTS_PROC_GET_SCHDATA((p))->virtual_reds += vreds;
     }
-    return ERTS_ABANDON_HEAP_COST;
+
+    ASSERT(CONTEXT_REDS >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+    return reds_left;
 }
 
 static ERTS_FORCE_INLINE Uint
@@ -570,7 +574,7 @@ young_gen_usage(Process *p)
  */
 static int
 garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
-		int need, Eterm* objv, int nobj)
+		int need, Eterm* objv, int nobj, int fcalls)
 {
     Uint reclaimed_now = 0;
     int reds;
@@ -581,8 +585,11 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
 
+    ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
+	   >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+
     if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
-	return delay_garbage_collection(p, live_hf_end, need);
+	return delay_garbage_collection(p, live_hf_end, need, fcalls);
 
     if (p->abandoned_heap)
 	live_hf_end = ERTS_INVALID_HFRAG_PTR;
@@ -703,16 +710,23 @@ do_major_collection:
 }
 
 int
-erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj)
+erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls)
 {
-    return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls);
+    int reds_left = ERTS_REDS_LEFT(p, fcalls);
+    if (reds > reds_left)
+	reds = reds_left;
+    ASSERT(CONTEXT_REDS - (reds_left - reds) >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
+    return reds;
 }
 
 void
 erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 {
-    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls);
     BUMP_REDS(p, reds);
+    ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p)
+	   >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds);
 }
 
 /*
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 40b7c5d12c..8a6ff99990 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -139,7 +139,7 @@ Eterm erts_process_gc_info(struct process*, Uint *, Eterm **);
 
 void erts_gc_info(ErtsGCInfo *gcip);
 void erts_init_gc(void);
-int erts_garbage_collect_nobump(struct process*, int, Eterm*, int);
+int erts_garbage_collect_nobump(struct process*, int, Eterm*, int, int);
 void erts_garbage_collect(struct process*, int, Eterm*, int);
 void erts_garbage_collect_hibernate(struct process* p);
 Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end,
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 2fd1eeb785..fcd2739ac3 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -206,15 +206,15 @@ int erts_no_line_info = 0;	/* -L: Don't load line information */
 
 ErtsModifiedTimings erts_modified_timings[] = {
     /* 0 */	{make_small(0), CONTEXT_REDS, INPUT_REDUCTIONS},
-    /* 1 */	{make_small(0), 2*CONTEXT_REDS, 2*INPUT_REDUCTIONS},
+    /* 1 */	{make_small(0), (3*CONTEXT_REDS)/4, 2*INPUT_REDUCTIONS},
     /* 2 */	{make_small(0), CONTEXT_REDS/2, INPUT_REDUCTIONS/2},
-    /* 3 */	{make_small(0), 3*CONTEXT_REDS, 3*INPUT_REDUCTIONS},
+    /* 3 */	{make_small(0), (7*CONTEXT_REDS)/8, 3*INPUT_REDUCTIONS},
     /* 4 */	{make_small(0), CONTEXT_REDS/3, 3*INPUT_REDUCTIONS},
-    /* 5 */	{make_small(0), 4*CONTEXT_REDS, INPUT_REDUCTIONS/2},
+    /* 5 */	{make_small(0), (10*CONTEXT_REDS)/11, INPUT_REDUCTIONS/2},
     /* 6 */	{make_small(1), CONTEXT_REDS/4, 2*INPUT_REDUCTIONS},
-    /* 7 */	{make_small(1), 5*CONTEXT_REDS, INPUT_REDUCTIONS/3},
+    /* 7 */	{make_small(1), (5*CONTEXT_REDS)/7, INPUT_REDUCTIONS/3},
     /* 8 */	{make_small(10), CONTEXT_REDS/5, 3*INPUT_REDUCTIONS},
-    /* 9 */	{make_small(10), 6*CONTEXT_REDS, INPUT_REDUCTIONS/4}
+    /* 9 */	{make_small(10), (6*CONTEXT_REDS)/7, INPUT_REDUCTIONS/4}
 };
 
 #define ERTS_MODIFIED_TIMING_LEVELS \
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index d485affa3b..c6c3fd64a1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9240,6 +9240,21 @@ erts_set_process_priority(Process *p, Eterm value)
     }
 }
 
+static int
+scheduler_gc_proc(Process *c_p, int reds_left)
+{
+    int fcalls, reds;
+    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	fcalls = reds_left;
+    else
+	fcalls = reds_left - CONTEXT_REDS;
+    reds = erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls);
+    ASSERT(reds_left >= reds);
+    return reds;
+}
+
+
+
 /*
  * schedule() is called from BEAM (process_main()) or HiPE
  * (hipe_mode_switch()) when the current process is to be
@@ -9318,6 +9333,7 @@ Process *schedule(Process *p, int calls)
 #endif
 
 	reds = actual_reds = calls - esdp->virtual_reds;
+	ASSERT(actual_reds >= 0);
 	if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST)
 	    reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST;
 	esdp->virtual_reds = 0;
@@ -9740,7 +9756,7 @@ Process *schedule(Process *p, int calls)
 
 	    esdp->current_process = p;
 
-
+	    calls = 0;
 	    reds = context_reds;
 
 #ifdef ERTS_SMP
@@ -9819,9 +9835,7 @@ Process *schedule(Process *p, int calls)
 		/* Migrate to dirty scheduler... */
 	    sunlock_sched_out_proc:
 		erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
-		p->fcalls = reds;
 		goto sched_out_proc;
-
 	    }
 	}
 	else {
@@ -9856,8 +9870,6 @@ Process *schedule(Process *p, int calls)
 
 #endif /* ERTS_SMP */
 
-	p->fcalls = reds;
-
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
 
 	if (IS_TRACED(p)) {
@@ -9890,14 +9902,15 @@ Process *schedule(Process *p, int calls)
 	     * not allowed to execute system tasks.
 	     */
 	    if (!(p->flags & F_DELAY_GC)) {
-		reds -= execute_sys_tasks(p, &state, reds);
+		int cost = execute_sys_tasks(p, &state, reds);
+		calls += cost;
+		reds -= cost;
 		if (reds <= 0
 #ifdef ERTS_DIRTY_SCHEDULERS
 		    || ERTS_SCHEDULER_IS_DIRTY(esdp)
 		    || (state & ERTS_PSFLGS_DIRTY_WORK)
 #endif
 		    ) {
-		    p->fcalls = reds;
 		    goto sched_out_proc;
 		}
 	    }
@@ -9910,8 +9923,9 @@ Process *schedule(Process *p, int calls)
 
 		if (((state & (ERTS_PSFLG_SUSPENDED
 			       | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
-		    && !(state & ERTS_PSFLG_EXITING))
+		    && !(state & ERTS_PSFLG_EXITING)) {
 		    goto sched_out_proc;
+		}
 
 		n = e = state;
 		n &= ~ERTS_PSFLG_RUNNING_SYS;
@@ -9930,11 +9944,11 @@ Process *schedule(Process *p, int calls)
 
 	if (ERTS_IS_GC_DESIRED(p)) {
 	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
-		reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity);
-		if (reds <= 0) {
-		    p->fcalls = reds;
+		int cost = scheduler_gc_proc(p, reds);
+		calls += cost;
+		reds -= cost;
+		if (reds <= 0)
 		    goto sched_out_proc;
-		}
 	    }
 	}
 
@@ -9942,6 +9956,8 @@ Process *schedule(Process *p, int calls)
 	    free_proxy_proc(proxy_p);
 	    proxy_p = NULL;
 	}
+
+	p->fcalls = reds;
 	    
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
@@ -10211,21 +10227,24 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	    else {
 		if (!garbage_collected) {
 		    FLAGS(c_p) |= F_NEED_FULLSWEEP;
-		    reds -= erts_garbage_collect_nobump(c_p,
-							0,
-							c_p->arg_reg,
-							c_p->arity);
+		    reds -= scheduler_gc_proc(c_p, reds);
 		    garbage_collected = 1;
 		}
 		st_res = am_true;
 	    }
 	    break;
 	case ERTS_PSTT_CPC: {
+	    int fcalls;
             int cpc_reds = 0;
+	    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+		fcalls = reds;
+	    else
+		fcalls = reds - CONTEXT_REDS;
 	    st_res = erts_check_process_code(c_p,
 					     st->arg[0],
                                              unsigned_val(st->arg[1]),
-					     &cpc_reds);
+					     &cpc_reds,
+					     fcalls);
             reds -= cpc_reds;
 	    if (is_non_value(st_res)) {
 		/* Needed gc, but gc was disabled */
@@ -10257,6 +10276,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 
     *statep = state;
 
+    if (in_reds < reds)
+	return in_reds;
+
     return in_reds - reds;
 }
 
@@ -11482,8 +11504,10 @@ delete_process(Process* p)
 
     psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd);
 
-    if (psd)
+    if (psd) {
+	erts_smp_atomic_set_nob(&p->psd, (erts_aint_t) NULL); /* Reduction counting depends on this... */
 	erts_free(ERTS_ALC_T_PSD, psd);
+    }
 
     /* Clean binaries and funs */
     erts_cleanup_offheap(&p->off_heap);
@@ -12577,6 +12601,8 @@ erts_continue_exit_process(Process *p)
     
     dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL;
     scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
+    if (scb)
+	p->fcalls += CONTEXT_REDS; /* Reduction counting depends on this... */
     pbt = ERTS_PROC_SET_CALL_TIME(p, NULL);
     nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 1c01d705a7..59da9c1779 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -805,8 +805,15 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
 #define ERTS_PSD_CALL_TIME_BP			3
 #define ERTS_PSD_DELAYED_GC_TASK_QS		4
 #define ERTS_PSD_NIF_TRAP_EXPORT		5
+#ifdef HIPE
+#define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	6
+#endif
 
+#ifdef HIPE
+#define ERTS_PSD_SIZE				7
+#else
 #define ERTS_PSD_SIZE				6
+#endif
 
 typedef struct {
     void *data[ERTS_PSD_SIZE];
@@ -1080,6 +1087,10 @@ struct process {
     Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
     Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
 #endif
+
+#ifdef DEBUG
+    Uint debug_reds_in;
+#endif
 };
 
 extern const Process erts_invalid_process;
@@ -1351,6 +1362,7 @@ extern int erts_system_profile_ts_type;
 #define F_DELAY_GC           (1 << 16) /* Similar to disable GC (see below) */
 #define F_SCHDLR_ONLN_WAITQ  (1 << 17) /* Process enqueued waiting to change schedulers online */
 #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */
+#define F_HIPE_MODE          (1 << 19)
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -2008,6 +2020,13 @@ erts_psd_set(Process *p, int ix, void *data)
 #define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \
     erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))
 
+#ifdef HIPE
+#define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \
+  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF))
+#define ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(P, SCB) \
+  ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB)))
+#endif
+
 ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
 ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
 
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 128696270b..436b4aca21 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1322,16 +1322,18 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid,
 
 void save_calls(Process *p, Export *e)
 {
-    struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
-    if (scb) {
-	Export **ct = &scb->ct[0];
-	int len = scb->len;
-
-	ct[scb->cur] = e;
-	if (++scb->cur >= len)
-	    scb->cur = 0;
-	if (scb->n < len)
-	    scb->n++;
+    if ((ERTS_TRACE_FLAGS(p) & F_SENSITIVE) == 0) {
+	struct saved_calls *scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
+	if (scb) {
+	    Export **ct = &scb->ct[0];
+	    int len = scb->len;
+
+	    ct[scb->cur] = e;
+	    if (++scb->cur >= len)
+		scb->cur = 0;
+	    if (scb->n < len)
+		scb->n++;
+	}
     }
 }
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index a7bc990deb..49eec44053 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -997,7 +997,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
 #define ERTS_CPC_ALLOW_GC      (1 << 0)
 #define ERTS_CPC_COPY_LITERALS (1 << 1)
 #define ERTS_CPC_ALL           (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS)
-Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp);
+Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls);
 
 typedef struct {
     Eterm *ptr;
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 43144e75ec..9ad44b25ac 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -218,15 +218,37 @@ static __inline__ void hipe_pop_beam_trap_frame(Process *p)
 Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 {
     unsigned result;
-#if NR_ARG_REGS > 5
-    /* When NR_ARG_REGS > 5, we need to protect the process' input
-       reduction count (which BEAM stores in def_arg_reg[5]) from
-       being clobbered by the arch glue code. */
     Eterm reds_in = p->def_arg_reg[5];
-#endif
-#if NR_ARG_REGS > 4
-    Eterm o_reds = p->def_arg_reg[4];
-#endif
+    /*
+     * Process is in the normal case scheduled out when reduction
+     * count reach zero. When "save calls" is enabled reduction
+     * count is subtracted with CONTEXT_REDS, i.e. initial reduction
+     * count will be zero or less and process is scheduled out
+     * when -CONTEXT_REDS is reached.
+     *
+     * HiPE does not support the "save calls" feature, so we switch
+     * to using a positive reduction counter when executing in
+     * hipe mode, but need to restore the "save calls" when
+     * returning to beam. We also need to hide the save calls buffer
+     * from BIFs. We do that by moving the saved calls buf to
+     * suspended saved calls buf.
+     *
+     * Beam has initial reduction count in stored in p->def_arg_reg[5].
+     *
+     * Beam expects -neg_o_reds to be found in p->def_arg_reg[4]
+     * on return to beam.
+     */
+
+    {
+	struct saved_calls *scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
+	if (scb) {
+	    reds_in += CONTEXT_REDS;
+	    p->fcalls += CONTEXT_REDS;
+	    ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, scb);
+	}
+    }
+
+    p->flags |= F_HIPE_MODE; /* inform bifs where we are comming from... */
 
     p->i = NULL;
     /* Set current_function to undefined. stdlib hibernate tests rely on it. */
@@ -481,12 +503,21 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 	  erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_MSG_RECEIVE);
       do_schedule:
 	  {
-#if !(NR_ARG_REGS > 5)
-	      int reds_in = p->def_arg_reg[5];
+	      struct saved_calls *scb;
+
+	      scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, NULL);
+	      if (scb)
+		  ERTS_PROC_SET_SAVED_CALLS_BUF(p, scb);
+
+#ifdef DEBUG
+	      ASSERT(p->debug_reds_in == reds_in);
 #endif
+	      p->flags &= ~F_HIPE_MODE;
+
 	      ERTS_SMP_UNREQ_PROC_MAIN_LOCK(p);
 	      p = schedule(p, reds_in - p->fcalls);
 	      ERTS_SMP_REQ_PROC_MAIN_LOCK(p);
+	      ASSERT(!(p->flags & F_HIPE_MODE));
 #ifdef ERTS_SMP
 	      p->hipe_smp.have_receive_locks = 0;
 	      reg = p->scheduler_data->x_reg_array;
@@ -501,28 +532,32 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
 		  reg[i] = argp[i];
 	  }
 	  {
-#if !(NR_ARG_REGS > 5)
-	      Eterm reds_in;
-#endif
-#if !(NR_ARG_REGS > 4)
-	      Eterm o_reds;
-#endif
+	      struct saved_calls *scb;
 
 	      reds_in = p->fcalls;
-	      o_reds = 0;
-	      if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
-		  o_reds = reds_in;
-		  reds_in = 0;
-		  p->fcalls = 0;
-	      }
-	      p->def_arg_reg[4] = o_reds;
 	      p->def_arg_reg[5] = reds_in;
+#ifdef DEBUG
+	      p->debug_reds_in = reds_in;
+#endif
 	      if (p->i == hipe_beam_pc_resume) {
+		  p->flags |= F_HIPE_MODE; /* inform bifs where we are comming from... */
+		  scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL);
+		  if (scb)
+		      ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, scb);
 		  p->i = NULL;
 		  p->arity = 0;
 		  goto do_resume;
 	      }
+
+	      scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p);
+	      if (!scb)
+		  p->def_arg_reg[4] = 0;
+	      else {
+		  p->def_arg_reg[4] = CONTEXT_REDS;
+		  p->fcalls = -CONTEXT_REDS + reds_in;
+	      }
 	  }
+
 	  HIPE_CHECK_PCB(p);
 	  result = HIPE_MODE_SWITCH_RES_CALL_BEAM;
 	  p->def_arg_reg[3] = result;
@@ -562,14 +597,29 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[])
       default:
 	erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %#x\r\n", result);
     }
+
+    {
+	struct saved_calls *scb = ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(p, NULL);
+	if (!scb)
+	    p->def_arg_reg[4] = 0;
+	else {
+	    p->def_arg_reg[4] = CONTEXT_REDS;
+	    p->fcalls -= CONTEXT_REDS;
+	    ERTS_PROC_SET_SAVED_CALLS_BUF(p, scb);
+	}
+    }
+
     HIPE_CHECK_PCB(p);
     p->def_arg_reg[3] = result;
-#if NR_ARG_REGS > 4
-    p->def_arg_reg[4] = o_reds;
-#endif
 #if NR_ARG_REGS > 5
+    /*
+     * When NR_ARG_REGS > 5, we need to protect the process' input
+     * reduction count (which BEAM stores in def_arg_reg[5]) from
+     * being clobbered by the arch glue code.
+     */
     p->def_arg_reg[5] = reds_in;
 #endif
+    p->flags &= ~F_HIPE_MODE;
     return p;
 }
 
diff --git a/erts/emulator/test/save_calls_SUITE.erl b/erts/emulator/test/save_calls_SUITE.erl
index af1a0d35d6..aae7651f6d 100644
--- a/erts/emulator/test/save_calls_SUITE.erl
+++ b/erts/emulator/test/save_calls_SUITE.erl
@@ -60,7 +60,7 @@ dont_break_reductions(Config) when is_list(Config) ->
     RPS1 = reds_per_sched(0),
     RPS2 = reds_per_sched(20),
     Diff = abs(RPS1 - RPS2),
-    true = (Diff < (0.05 * RPS1)),
+    true = (Diff < (0.2 * RPS1)),
     ok.