Fix 18 exabyte memory allocation failure

The new_binary() function takes a size argument that is an
int. In the 64-bit emulator (sizeof(int) == 4, sizeof(Uint) == 8),
any sizes >= 0x8000000 become 0xffffffff80000000 and above and
triggers a memory allocation failure.

Change the type of the size argument to Uint, and change any
callers that cast the argument to an int.

Correction-by: Jon Meredith

6 files changed, 8 insertions, 7 deletions

diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 2855241b91..2406e0e810 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -267,7 +267,7 @@ erts_debug_disassemble_1(Process* p, Eterm addr)
 		   "unknown " HEXF "\n", instr);
 	code_ptr++;
     }
-    bin = new_binary(p, (byte *) dsbufp->str, (int) dsbufp->str_len);
+    bin = new_binary(p, (byte *) dsbufp->str, dsbufp->str_len);
     erts_destroy_tmp_dsbuf(dsbufp);
     hsz = 4+4;
     (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr);
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 4be869f269..99c98f9e72 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -56,7 +56,7 @@ erts_init_binary(void)
  */
 
 Eterm
-new_binary(Process *p, byte *buf, int len)
+new_binary(Process *p, byte *buf, Uint len)
 {
     ProcBin* pb;
     Binary* bptr;
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 4a717d7271..a7b5920425 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1545,7 +1545,7 @@ process_info_aux(Process *BIF_P,
     case am_backtrace: {
 	erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
 	erts_stack_dump(ERTS_PRINT_DSBUF, (void *) dsbufp, rp);
-	res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len);
+	res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len);
 	erts_destroy_tmp_dsbuf(dsbufp);
 	hp = HAlloc(BIF_P, 3);
 	break;
@@ -2074,7 +2074,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
 
 	ASSERT(dsbufp && dsbufp->str);
-	res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len);
+	res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len);
 	erts_destroy_info_dsbuf(dsbufp);
 	BIF_RET(res);
     } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) {
diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c
index d3e31da413..0b63ab9ba0 100644
--- a/erts/emulator/beam/erl_db_util.c
+++ b/erts/emulator/beam/erl_db_util.c
@@ -2162,7 +2162,8 @@ restart:
 	case matchProcessDump: {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0);
 	    print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p);
-	    *esp++ = new_binary(build_proc, (byte *)dsbufp->str, (int)dsbufp->str_len);
+	    *esp++ = new_binary(build_proc, (byte *)dsbufp->str,
+				dsbufp->str_len);
 	    erts_destroy_tmp_dsbuf(dsbufp);
 	    break;
 	}
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 328aa2be6a..b2cf685cf1 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -1264,7 +1264,7 @@ external_size_1(Process* p, Eterm Term)
 Eterm
 erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags)
 {
-    int size;
+    Uint size;
     Eterm bin;
     size_t real_size;
     byte* endp;
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index bd540eaaa6..a94ddf25be 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -834,7 +834,7 @@ do {									\
 void erts_emasculate_writable_binary(ProcBin* pb);
 Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap);
 Eterm erts_new_mso_binary(Process*, byte*, int);
-Eterm new_binary(Process*, byte*, int);
+Eterm new_binary(Process*, byte*, Uint);
 Eterm erts_realloc_binary(Eterm bin, size_t size);
 
 /* erl_bif_info.c */