erts: Fix size overflow bugs in memory allocation

3 files changed, 29 insertions, 13 deletions

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 45f0cc4312..a4e164bf51 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -3274,6 +3274,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags)
 
     ASSERT(!(flags & CFLG_FORCE_MSEG && flags & CFLG_FORCE_SYS_ALLOC));
 
+    if (umem_sz > (ERTS_UINT_MAX - ERTS_UINT_MAX/100)) {
+	/* Do an overly conservative _overflow_ check here so we don't
+	 * have to deal with it from here on. I guess we could be more accurate
+	 * but I don't think the need to allocate over 99% of the address space
+	 * will ever arise on any machine, neither 32 nor 64 bit.
+	 */
+	return NULL;
+    }
+
     blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz);
 
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index 6c9f53ce87..06dfeb1260 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -236,6 +236,8 @@ erts_bin_drv_alloc_fnf(Uint size)
 {
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
     void *res;
+    if (bsize < size) /* overflow */
+	return NULL;
     res = erts_alloc_fnf(ERTS_ALC_T_DRV_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
     return (Binary *) res;
@@ -246,6 +248,8 @@ erts_bin_drv_alloc(Uint size)
 {
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
     void *res;
+    if (bsize < size) /* overflow */
+	erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, size);
     res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
     return (Binary *) res;
@@ -257,6 +261,8 @@ erts_bin_nrml_alloc(Uint size)
 {
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
     void *res;
+    if (bsize < size) /* overflow */
+	erts_alloc_enomem(ERTS_ALC_T_BINARY, size);
     res = erts_alloc(ERTS_ALC_T_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
     return (Binary *) res;
@@ -267,11 +273,12 @@ erts_bin_realloc_fnf(Binary *bp, Uint size)
 {
     Binary *nbp;
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
+    ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
+	                                            : ERTS_ALC_T_BINARY;
     ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0);
-    if (bp->flags & BIN_FLAG_DRV)
-	nbp = erts_realloc_fnf(ERTS_ALC_T_DRV_BINARY, (void *) bp, bsize);
-    else
-	nbp = erts_realloc_fnf(ERTS_ALC_T_BINARY, (void *) bp, bsize);
+    if (bsize < size) /* overflow */
+	return NULL;
+    nbp = erts_realloc_fnf(type, (void *) bp, bsize);
     ERTS_CHK_BIN_ALIGNMENT(nbp);
     return nbp;
 }
@@ -281,17 +288,14 @@ erts_bin_realloc(Binary *bp, Uint size)
 {
     Binary *nbp;
     Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
+    ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
+	                                            : ERTS_ALC_T_BINARY;
     ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0);
-    if (bp->flags & BIN_FLAG_DRV)
-	nbp = erts_realloc_fnf(ERTS_ALC_T_DRV_BINARY, (void *) bp, bsize);
-    else
-	nbp = erts_realloc_fnf(ERTS_ALC_T_BINARY, (void *) bp, bsize);
+    if (bsize < size) /* overflow */
+	erts_realloc_enomem(type, bp, size);
+    nbp = erts_realloc_fnf(type, (void *) bp, bsize);
     if (!nbp)
-	erts_realloc_n_enomem(ERTS_ALC_T2N(bp->flags & BIN_FLAG_DRV
-					   ? ERTS_ALC_T_DRV_BINARY
-					   : ERTS_ALC_T_BINARY),
-			      bp,
-			      bsize);
+	erts_realloc_enomem(type, bp, bsize);
     ERTS_CHK_BIN_ALIGNMENT(nbp);
     return nbp;
 }
@@ -312,6 +316,7 @@ erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
 {
     Uint bsize = ERTS_MAGIC_BIN_SIZE(size);
     Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize);
+    ASSERT(bsize > size);
     if (!bptr)
 	erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize);
     ERTS_CHK_BIN_ALIGNMENT(bptr);
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 05f07e57b2..3d8dd9c6d0 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -274,6 +274,7 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f
 typedef unsigned int Eterm;
 typedef unsigned int Uint;
 typedef int          Sint;
+#define ERTS_UINT_MAX UINT_MAX
 #define ERTS_SIZEOF_ETERM SIZEOF_INT
 #define ErtsStrToSint strtol
 #else
@@ -347,6 +348,7 @@ typedef long long          Sint;
 
 typedef Uint UWord;
 typedef Sint SWord;
+#define ERTS_UINT_MAX ERTS_UWORD_MAX
 
 #endif /* HALFWORD_HEAP */