Increase vheap counter to Uint64

This will reduce the risk of integer wrapping in bin vheap counting.

The vheap size series will now use the golden ratio instead of doubling
and fibonacci sequences.

OTP #8730

12 files changed, 60 insertions, 53 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 260f349563..0b29644f24 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3390,7 +3390,7 @@ apply_bif_or_nif_epilogue:
 	 pb->val = bptr;
 	 pb->bytes = (byte*) bptr->orig_bytes;
 	 pb->flags = 0;
-	 MSO(c_p).overhead += pb->size / sizeof(Eterm);
+	 OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
 	 new_binary = make_binary(pb);
 	 goto do_bits_sub_bin;
      }
@@ -3492,7 +3492,7 @@ apply_bif_or_nif_epilogue:
 	 pb->bytes = (byte*) bptr->orig_bytes;
 	 pb->flags = 0;
 	 
-	 MSO(c_p).overhead += tmp_arg1 / sizeof(Eterm);
+	 OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm));
 
 	 StoreBifResult(2, make_binary(pb));
      }
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 3fd714f9c2..8ee8fbcb29 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -97,7 +97,7 @@ new_binary(Process *p, byte *buf, int len)
     /*
      * Miscellanous updates. Return the tagged binary.
      */
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
 }
 
@@ -136,7 +136,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len)
     /*
      * Miscellanous updates. Return the tagged binary.
      */
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     return make_binary(pb);
 }
 
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c
index 57be0169ba..8bee47232e 100644
--- a/erts/emulator/beam/copy.c
+++ b/erts/emulator/beam/copy.c
@@ -317,7 +317,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		    pb->next = off_heap->first;
 		    pb->flags = 0;
 		    off_heap->first = (struct erl_off_heap_header*) pb;
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
 		break;
 	    case SUB_BINARY_SUBTAG:
@@ -366,7 +366,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 			to->next = off_heap->first;
 			to->flags = 0;
 			off_heap->first = (struct erl_off_heap_header*) to;
-			off_heap->overhead += to->size / sizeof(Eterm);
+			OH_OVERHEAD(off_heap, to->size / sizeof(Eterm));
 		    }
 		    *argp = make_binary(hbot);
 		    if (extra_bytes != 0) {
@@ -652,7 +652,7 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                 erts_refc_inc(&pb->val->refc, 2);
                 pb->next = erts_global_offheap.first;
                 erts_global_offheap.first = pb;
-                erts_global_offheap.overhead += pb->size / sizeof(Eterm);
+		OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
                 continue;
             }
 
@@ -777,7 +777,7 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs)
                     to_bin->bytes = from_bin->bytes + sub_offset;
                     to_bin->next = erts_global_offheap.first;
                     erts_global_offheap.first = to_bin;
-                    erts_global_offheap.overhead += to_bin->size / sizeof(Eterm);
+		    OH_OVERHEAD(&erts_global_offheap, to_bin->size / sizeof(Eterm));
 		    res_binary=make_binary(to_bin);
 		    hp += PROC_BIN_SIZE;
                 }
@@ -912,7 +912,7 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap)
 		{
 		    ProcBin* pb = (ProcBin *) (tp-1);
 		    erts_refc_inc(&pb->val->refc, 2);
-		    off_heap->overhead += pb->size / sizeof(Eterm);
+		    OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm));
 		}
 		goto off_heap_common;
 
@@ -977,7 +977,7 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first,
 
     for (bp=first; bp!=NULL; bp=bp->next) {
 	move_one_frag(hpp, bp->mem, bp->used_size, off_heap);
-	off_heap->overhead += bp->off_heap.overhead;
+	OH_OVERHEAD(off_heap, bp->off_heap.overhead);
     }
     hp_end = *hpp;
     for (hp=hp_start; hp<hp_end; ++hp) {
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 1d8fd11b7b..59fce04343 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -2517,7 +2517,7 @@ BIF_RETTYPE binary_copy_trap(BIF_ALIST_2)
 	pb->bytes = t;
 	pb->flags = 0;
 
-	MSO(BIF_P).overhead += target_size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm));
 	BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR);
 
 	BIF_RET(make_binary(pb));
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 96faa673f6..88d2c06246 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1340,7 +1340,7 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 	pb->val = bptr;
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
-	MSO(c_p).overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
 
 	/*
 	 * Now allocate the sub binary and set its size to include the
@@ -1511,7 +1511,7 @@ erts_bs_init_writable(Process* p, Eterm sz)
     pb->val = bptr;
     pb->bytes = (byte*) bptr->orig_bytes;
     pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER;
-    MSO(p).overhead += pb->size / sizeof(Eterm);
+    OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm));
     
     /*
      * Now allocate the sub binary.
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 38cc6195fb..0f4d2a2ef9 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -743,7 +743,10 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
      * is large enough.
      */
 
-    if (OLD_HEAP(p) && mature <= OLD_HEND(p) - OLD_HTOP(p)) {
+    if (OLD_HEAP(p) &&
+	    ((mature <= OLD_HEND(p) - OLD_HTOP(p)) &&
+	    ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
+	    ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
 	ErlMessage *msgp;
 	Uint size_after;
 	Uint need_after;
@@ -1982,8 +1985,8 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj)
     HEAP_SIZE(p) = new_sz;
 }
 
-static Uint
-do_next_vheap_size(Uint vheap, Uint vheap_sz) {
+static Uint64
+do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) {
 
     /*                grow
      *
@@ -1999,37 +2002,34 @@ do_next_vheap_size(Uint vheap, Uint vheap_sz) {
      *
      *          ----------------------
      */
-    Uint vheap_max = heap_sizes[num_heap_sizes - 1];
-    if (vheap_sz == vheap_max) {
-	return vheap_sz;
-    }
 
-    if ((Uint) vheap/3 > (Uint) (vheap_sz/4)) {
-	Uint new_vheap_sz = vheap_sz;
+    if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+	Uint64 new_vheap_sz = vheap_sz;
 
-	while((Uint) vheap/3 > (Uint) (vheap_sz/4)) {
-	    new_vheap_sz = (vheap_sz << 1);
-	    if (new_vheap_sz < vheap_sz || new_vheap_sz > vheap_max ) {
-		return vheap_max;
+	while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+	    /* the golden ratio = 1.618 */
+	    new_vheap_sz = (Uint64) vheap_sz * 1.618;
+	    if (new_vheap_sz < vheap_sz ) {
+	        return vheap_sz;
 	    }
 	    vheap_sz = new_vheap_sz;
 	}
 
-	return erts_next_heap_size(vheap_sz, 0);
+	return vheap_sz;
     }
 
-    if (vheap < (Uint) (vheap_sz/4)) {
-	return erts_next_heap_size((Uint) (vheap_sz >> 1), 0);
+    if (vheap < (Uint64) (vheap_sz/4)) {
+	return (vheap_sz >> 1);
     }
 
     return vheap_sz;
 
 }
 
-static Uint
-next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) {
-    vheap_sz = do_next_vheap_size(vheap, vheap_sz);
-    return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz;
+static Uint64
+next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) {
+    Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz);
+    return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz;
 }
 
 struct shrink_cand_data {
@@ -2088,7 +2088,7 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
 }
 
 
-static void 
+static void
 sweep_off_heap(Process *p, int fullsweep)
 {
     struct shrink_cand_data shrink = {0};
@@ -2096,7 +2096,7 @@ sweep_off_heap(Process *p, int fullsweep)
     struct erl_off_heap_header** prev;
     char* oheap = NULL;
     Uint oheap_sz = 0;
-    Uint bin_vheap = 0;
+    Uint64 bin_vheap = 0;
 #ifdef DEBUG
     int seen_mature = 0;
 #endif
@@ -2180,13 +2180,12 @@ sweep_off_heap(Process *p, int fullsweep)
 	}
     }
 
-    if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) {
-        FLAGS(p) |= F_NEED_FULLSWEEP;
+    if (fullsweep) {
+	BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p));
     }
-
-    BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
-    BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p));
-    MSO(p).overhead = bin_vheap;
+    BIN_VHEAP_SZ(p)     = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
+    MSO(p).overhead     = bin_vheap;
+    BIN_VHEAP_MATURE(p) = bin_vheap;
 
     /*
      * If we got any shrink candidates, check them out.
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index 1f61dca230..82f272d28a 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -220,7 +220,7 @@ link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp)
 	    *next_p = MSO(proc).first;
 	    MSO(proc).first = bp->off_heap.first;
 	    bp->off_heap.first = NULL;
-	    MSO(proc).overhead += bp->off_heap.overhead;
+	    OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead);
 	}
     }
 }
@@ -535,7 +535,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg)
 	goto copy_done;
     }
 
-    off_heap->overhead += bp->off_heap.overhead;
+    OH_OVERHEAD(off_heap, bp->off_heap.overhead);
     sz = bp->used_size;
 
     ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp));
diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h
index 55ea92860a..5aca0db6fe 100644
--- a/erts/emulator/beam/erl_message.h
+++ b/erts/emulator/beam/erl_message.h
@@ -37,9 +37,13 @@ struct erl_off_heap_header {
     struct erl_off_heap_header* next;
 };
 
+#define OH_OVERHEAD(oh, size) do { \
+    (oh)->overhead += size;        \
+} while(0)
+
 typedef struct erl_off_heap {
     struct erl_off_heap_header* first;
-    int overhead;     /* Administrative overhead (used to force GC). */
+    Uint64 overhead;     /* Administrative overhead (used to force GC). */
 } ErlOffHeap;
 
 #include "external.h"
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index e95d9c4f75..eadaf137e3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -631,7 +631,7 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)
 	pb->bytes = (byte*) bptr->orig_bytes;
 	pb->flags = 0;
 	
-	MSO(env->proc).overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm));
 	bin_term = make_binary(pb);	
 	if (erts_refc_read(&bptr->refc, 1) == 1) {
 	    /* Total ownership transfer */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index a644520442..c2e62f56a5 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7799,6 +7799,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->bin_vheap_sz     = p->min_vheap_size;
     p->bin_old_vheap_sz = p->min_vheap_size;
     p->bin_old_vheap    = 0;
+    p->bin_vheap_mature = 0;
 
     /* No need to initialize p->fcalls. */
 
@@ -8051,6 +8052,7 @@ void erts_init_empty_process(Process *p)
     p->bin_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap_sz = BIN_VH_MIN_SIZE;
     p->bin_old_vheap = 0;
+    p->bin_vheap_mature = 0;
 #ifdef ERTS_SMP
     p->u.ptimer = NULL;
     p->bound_runq = NULL;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index c3fef6d38e..08b0144c41 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -537,6 +537,7 @@ struct ErtsPendingSuspend_ {
 
 #  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
 #  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
+#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
 #  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
 #  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap
 
@@ -654,9 +655,10 @@ struct process {
     Uint mbuf_sz;		/* Size of all message buffers */
     ErtsPSD *psd;		/* Rarely used process specific data */
 
-    Uint bin_vheap_sz;		/* Virtual heap block size for binaries */
-    Uint bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
-    Uint bin_old_vheap;		/* Virtual old heap size for binaries */
+    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
+    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
+    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
+    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
     union {
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index b840f65cdd..79022d5dd7 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -1582,7 +1582,7 @@ static void deliver_read_message(Port* prt, Eterm to,
 	pb->flags = 0;
 	hp += PROC_BIN_SIZE;
 
-	ohp->overhead += pb->size / sizeof(Eterm);
+	OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
 	listp = make_binary(pb);
     }
 
@@ -1732,7 +1732,7 @@ deliver_vec_message(Port* prt,			/* Port */
 	    pb->flags = 0;
 	    hp += PROC_BIN_SIZE;
 	    
-	    ohp->overhead += iov->iov_len / sizeof(Eterm);
+	    OH_OVERHEAD(ohp, iov->iov_len / sizeof(Eterm));
 
 	    if (listp == NIL) {  /* compatible with deliver_bin_message */
 		listp = make_binary(pb);
@@ -2264,7 +2264,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
 		pb->val = ErlDrvBinary2Binary(dbin);
 		pb->bytes = (byte*) dbin->orig_bytes;
 		pb->flags = 0;
-		MSO(p).overhead += dbin->orig_size / sizeof(Eterm);
+		OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm));
 		return make_binary(pb);
 	    }
 	    port_resp = dbin->orig_bytes;
@@ -3040,7 +3040,7 @@ driver_deliver_term(ErlDrvPort port,
 		pb->flags = 0;
 		mess =  make_binary(pb);
 		hp += PROC_BIN_SIZE;
-		ohp->overhead += pb->size / sizeof(Eterm);
+		OH_OVERHEAD(ohp, pb->size / sizeof(Eterm));
 	    }
 	    ptr += 3;
 	    break;
@@ -3077,7 +3077,7 @@ driver_deliver_term(ErlDrvPort port,
 		pbp->val = bp;
 		pbp->bytes = (byte*) bp->orig_bytes;
 		pbp->flags = 0;
-		ohp->overhead += (pbp->size / sizeof(Eterm));
+		OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm));
 		mess = make_binary(pbp);
 	    }
 	    ptr += 2;