Merge branch 'egil/R14A/binary-gc-wrap/OTP-8730' into dev

* egil/R14A/binary-gc-wrap/OTP-8730:
  Increase vheap counter to Uint64
  Fix wrapping in next vheap calculation

12 files changed, 62 insertions, 46 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 260f349563..0b29644f24 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3390,7 +3390,7 @@ apply_bif_or_nif_epilogue:
 	 pb->val = bptr;
 	 pb->bytes = (byte*) bptr->orig_bytes;
 	 pb->flags = 0;
-	 MSO(c_p).overhead += pb->size / sizeof(Eterm);
+	 OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
 	 new_binary = make_binary(pb);
 	 goto do_bits_sub_bin;
      }
@@ -3492,7 +3492,7 @@ apply_bif_or_nif_epilogue:
 	 pb->bytes = (byte*) bptr->orig_bytes;
 	 pb->flags = 0;
 	 
-	 MSO(c_p).overhead += tmp_arg1 / sizeof(Eterm);
+	 OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm));
 
 	 StoreBifResult(2, make_binary(pb));
      }
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 3fd714f9c2..8ee8fbcb29 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -97,7 +97,7 @@ new_binary(Process *p, byte *buf, int len)
     /*
      * Miscellanous updates. Return the tagged binary.
      */
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
 }
 
@@ -136,7 +136,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
     /*
      * Miscellanous updates. Return the tagged binary.
      */
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
 }
 
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 57be0169ba..8bee47232e 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -317,7 +317,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    pb->next = off_heap->first;
 		    pb->flags = 0;
 		    off_heap->first = (struct erl_off_heap_header*) pb;
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
 		break;
 	    case SUB_BINARY_SUBTAG:
@@ -366,7 +366,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			to->next = off_heap->first;
 			to->flags = 0;
 			off_heap->first = (struct erl_off_heap_header*) to;
-			off_heap->overhead += to->size / sizeof(Eterm);
+			OH_OVERHEAD(off_heap, to->size / sizeof(Eterm));
 		    }
 		    *argp = make_binary(hbot);
 		    if (extra_bytes != 0) {
@@ -652,7 +652,7 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                 erts_refc_inc(&pb->val->refc, 2);
                 pb->next = erts_global_offheap.first;
                 erts_global_offheap.first = pb;
-                erts_global_offheap.overhead += pb->size / sizeof(Eterm);
+		OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
                 continue;
             }
 
@@ -777,7 +777,7 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     to_bin->bytes = from_bin->bytes + sub_offset;
                     to_bin->next = erts_global_offheap.first;
                     erts_global_offheap.first = to_bin;
-                    erts_global_offheap.overhead += to_bin->size / sizeof(Eterm);
+		    OH_OVERHEAD(&erts_global_offheap, to_bin->size / sizeof(Eterm));
 		    res_binary=make_binary(to_bin);
 		    hp += PROC_BIN_SIZE;
                 }
@@ -912,7 +912,7 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		{
 		    ProcBin* pb = (ProcBin *) (tp-1);
 		    erts_refc_inc(&pb->val->refc, 2);
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
 		goto off_heap_common;
 
@@ -977,7 +977,7 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 
     for (bp=first; bp!=NULL; bp=bp->next) {
 	move_one_frag(hpp, bp->mem, bp->used_size, off_heap);
-	off_heap->overhead += bp->off_heap.overhead;
+	OH_OVERHEAD(off_heap, bp->off_heap.overhead);
     }
     hp_end = *hpp;
     for (hp=hp_start; hp<hp_end; ++hp) {
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 1d8fd11b7b..59fce04343 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2517,7 +2517,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2)
 	pb->bytes = t;
 	pb->flags = 0;
 
-	MSO(BIF_P).overhead += target_size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm));
 	BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR);
 
 	BIF_RET(make_binary(pb));
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 96faa673f6..88d2c06246 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1340,7 +1340,7 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 	pb->val = bptr;
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
-	MSO(c_p).overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
 
 	/*
 	 * Now allocate the sub binary and set its size to include the
@@ -1511,7 +1511,7 @@ erts_bs_init_writable(Process* p, Eterm sz)
     pb->val = bptr;
     pb->bytes = (byte*) bptr->orig_bytes;
     pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     
     /*
      * Now allocate the sub binary.
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index adc50675bf..0f4d2a2ef9 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -743,7 +743,10 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
      * is large enough.
      */
 
-    if (OLD_HEAP(p) && mature <= OLD_HEND(p) - OLD_HTOP(p)) {
+    if (OLD_HEAP(p) &&
+	    ((mature <= OLD_HEND(p) - OLD_HTOP(p)) &&
+	    ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
+	    ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
 	ErlMessage *msgp;
 	Uint size_after;
 	Uint need_after;
@@ -1982,8 +1985,8 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj)
     HEAP_SIZE(p) = new_sz;
 }
 
-static Uint
-do_next_vheap_size(Uint vheap, Uint vheap_sz) {
+static Uint64
+do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) {
 
     /*                grow
      *
@@ -2000,27 +2003,33 @@ do_next_vheap_size(Uint vheap, Uint vheap_sz) {
      *          ----------------------
      */
 
-    if (vheap > (Uint) (vheap_sz*3/4)) {
+    if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+	Uint64 new_vheap_sz = vheap_sz;
 
-	while(vheap > (Uint) (vheap_sz*3/4)) {
-	    vheap_sz = vheap_sz*2;
+	while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+	    /* the golden ratio = 1.618 */
+	    new_vheap_sz = (Uint64) vheap_sz * 1.618;
+	    if (new_vheap_sz < vheap_sz ) {
+	        return vheap_sz;
+	    }
+	    vheap_sz = new_vheap_sz;
 	}
 
-	return erts_next_heap_size(vheap_sz, 0);
+	return vheap_sz;
     }
 
-    if (vheap < (Uint) (vheap_sz/4)) {
-	return erts_next_heap_size((Uint) (vheap_sz / 2), 0);
+    if (vheap < (Uint64) (vheap_sz/4)) {
+	return (vheap_sz >> 1);
     }
 
     return vheap_sz;
 
 }
 
-static Uint
-next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) {
-    vheap_sz = do_next_vheap_size(vheap, vheap_sz);
-    return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz;
+static Uint64
+next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) {
+    Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz);
+    return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz;
 }
 
 struct shrink_cand_data {
@@ -2079,7 +2088,7 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
 }
 
 
-static void 
+static void
 sweep_off_heap(Process *p, int fullsweep)
 {
     struct shrink_cand_data shrink = {0};
@@ -2087,7 +2096,7 @@ sweep_off_heap(Process *p, int fullsweep)
     struct erl_off_heap_header** prev;
     char* oheap = NULL;
     Uint oheap_sz = 0;
-    Uint bin_vheap = 0;
+    Uint64 bin_vheap = 0;
 #ifdef DEBUG
     int seen_mature = 0;
 #endif
@@ -2171,13 +2180,12 @@ sweep_off_heap(Process *p, int fullsweep)
 	}
     }
 
-    if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) {
-        FLAGS(p) |= F_NEED_FULLSWEEP;
+    if (fullsweep) {
+	BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p));
     }
-
-    BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
-    BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p));
-    MSO(p).overhead = bin_vheap;
+    BIN_VHEAP_SZ(p)     = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
+    MSO(p).overhead     = bin_vheap;
+    BIN_VHEAP_MATURE(p) = bin_vheap;
 
     /*
      * If we got any shrink candidates, check them out.
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 1f61dca230..82f272d28a 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -220,7 +220,7 @@ link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp)
 	    *next_p = MSO(proc).first;
 	    MSO(proc).first = bp->off_heap.first;
 	    bp->off_heap.first = NULL;
-	    MSO(proc).overhead += bp->off_heap.overhead;
+	    OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
 	}
     }
 }
@@ -535,7 +535,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 	goto copy_done;
     }
 
-    off_heap->overhead += bp->off_heap.overhead;
+    OH_OVERHEAD(off_heap, bp->off_heap.overhead);
     sz = bp->used_size;
 
     ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp));
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 55ea92860a..5aca0db6fe 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -37,9 +37,13 @@ struct erl_off_heap_header {
     struct erl_off_heap_header* next;
 };
 
+#define OH_OVERHEAD(oh, size) do { \
+    (oh)->overhead += size;        \
+} while(0)
+
 typedef struct erl_off_heap {
     struct erl_off_heap_header* first;
-    int overhead;     /* Administrative overhead (used to force GC). */
+    Uint64 overhead;     /* Administrative overhead (used to force GC). */
 } ErlOffHeap;
 
 #include "external.h"
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e95d9c4f75..eadaf137e3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -631,7 +631,7 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = 0;
 	
-	MSO(env->proc).overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm));
 	bin_term = make_binary(pb);	
 	if (erts_refc_read(&bptr->refc, 1) == 1) {
 	    /* Total ownership transfer */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 0e78727316..5371153a79 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7840,6 +7840,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->bin_vheap_sz     = p->min_vheap_size;
     p->bin_old_vheap_sz = p->min_vheap_size;
     p->bin_old_vheap    = 0;
+    p->bin_vheap_mature = 0;
 
     /* No need to initialize p->fcalls. */
 
@@ -8092,6 +8093,7 @@ void erts_init_empty_process(Process *p)
     p->bin_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap = 0;
+    p->bin_vheap_mature = 0;
 #ifdef ERTS_SMP
     p->u.ptimer = NULL;
     p->bound_runq = NULL;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 0cc4a715ad..e49710a7ed 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -537,6 +537,7 @@ struct ErtsPendingSuspend_ {
 
 #  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
 #  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
+#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
 #  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
 #  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap
 
@@ -654,9 +655,10 @@ struct process {
     Uint mbuf_sz;		/* Size of all message buffers */
     ErtsPSD *psd;		/* Rarely used process specific data */
 
-    Uint bin_vheap_sz;		/* Virtual heap block size for binaries */
-    Uint bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
-    Uint bin_old_vheap;		/* Virtual old heap size for binaries */
+    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
+    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
+    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
+    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
     union {
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b840f65cdd..79022d5dd7 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1582,7 +1582,7 @@ static void deliver_read_message(Port* prt, Eterm to,
 	pb->flags = 0;
 	hp += PROC_BIN_SIZE;
 
-	ohp->overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
 	listp = make_binary(pb);
     }
 
@@ -1732,7 +1732,7 @@ deliver_vec_message(Port* prt,			/* Port */
 	    pb->flags = 0;
 	    hp += PROC_BIN_SIZE;
 	    
-	    ohp->overhead += iov->iov_len / sizeof(Eterm);
+	    OH_OVERHEAD(ohp, iov->iov_len / sizeof(Eterm));
 
 	    if (listp == NIL) {  /* compatible with deliver_bin_message */
 		listp = make_binary(pb);
@@ -2264,7 +2264,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
 		pb->val = ErlDrvBinary2Binary(dbin);
 		pb->bytes = (byte*) dbin->orig_bytes;
 		pb->flags = 0;
-		MSO(p).overhead += dbin->orig_size / sizeof(Eterm);
+		OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm));
 		return make_binary(pb);
 	    }
 	    port_resp = dbin->orig_bytes;
@@ -3040,7 +3040,7 @@ driver_deliver_term(ErlDrvPort port,
 		pb->flags = 0;
 		mess =  make_binary(pb);
 		hp += PROC_BIN_SIZE;
-		ohp->overhead += pb->size / sizeof(Eterm);
+		OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
 	    }
 	    ptr += 3;
 	    break;
@@ -3077,7 +3077,7 @@ driver_deliver_term(ErlDrvPort port,
 		pbp->val = bp;
 		pbp->bytes = (byte*) bp->orig_bytes;
 		pbp->flags = 0;
-		ohp->overhead += (pbp->size / sizeof(Eterm));
+		OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm));
 		mess = make_binary(pbp);
 	    }
 	    ptr += 2;