erts: Disallow binaries whose size in bits exceeds UWORD_MAX

These have never worked in binary matching (including sub-binaries
extracted from them) so it's hard to justify their existence. They
also make a future migration of binary sizes from bytes to bits
problematic, so we may as well change it ahead of time.

This is potentially incompatible on 32-bit platforms where a NIF
or driver could allocate 512MB+ binaries, but allocations that
large should be expected to fail anyway.

1 files changed, 76 insertions, 42 deletions

diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index c9c047255a..66b59ef1a3 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -351,6 +351,19 @@ erts_free_aligned_binary_bytes(byte* buf)
     erts_free_aligned_binary_bytes_extra(buf,ERTS_ALC_T_TMP);
 }
 
+/* A binary's size in bits must fit into a word for matching to work. We used
+ * to allow creating larger binaries than this, but they acted really strangely
+ * in Erlang code and were pretty much only usable in drivers and NIFs.
+ *
+ * This check also ensures, indirectly, that there won't be an overflow when
+ * the size is bumped by CHICKEN_PAD and the binary struct itself. */
+#define BINARY_OVERFLOW_CHECK(BYTE_SIZE) \
+    do { \
+         if (ERTS_UNLIKELY(BYTE_SIZE > ERTS_UWORD_MAX / CHAR_BIT)) { \
+             return NULL; \
+         } \
+    } while(0)
+
 /* Explicit extra bytes allocated to counter buggy drivers.
 ** These extra bytes where earlier (< R13B04) added by an alignment-bug
 ** in this code. Do we dare remove this in some major release (R14?) maybe?
@@ -364,86 +377,107 @@ erts_free_aligned_binary_bytes(byte* buf)
 ERTS_GLB_INLINE Binary *
 erts_bin_drv_alloc_fnf(Uint size)
 {
-    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
     Binary *res;
+    Uint bsize;
+
+    BINARY_OVERFLOW_CHECK(size);
+    bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
 
-    if (bsize < size) /* overflow */
-	return NULL;
     res = erts_alloc_fnf(ERTS_ALC_T_DRV_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
+
     if (res) {
-	res->orig_size = size;
-	res->intern.flags = BIN_FLAG_DRV;
+        res->orig_size = size;
+        res->intern.flags = BIN_FLAG_DRV;
         erts_refc_init(&res->intern.refc, 1);
     }
+
     return res;
 }
 
 ERTS_GLB_INLINE Binary *
 erts_bin_drv_alloc(Uint size)
 {
-    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
+    Binary *res = erts_bin_drv_alloc_fnf(size);
+
+    if (res) {
+        return res;
+    }
+
+    erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, size);
+}
+
+ERTS_GLB_INLINE Binary *
+erts_bin_nrml_alloc_fnf(Uint size)
+{
     Binary *res;
+    Uint bsize;
 
-    if (bsize < size) /* overflow */
-	erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, size);
-    res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize);
+    BINARY_OVERFLOW_CHECK(size);
+    bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
+
+    res = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize);
     ERTS_CHK_BIN_ALIGNMENT(res);
-    res->orig_size = size;
-    res->intern.flags = BIN_FLAG_DRV;
-    erts_refc_init(&res->intern.refc, 1);
+
+    if (res) {
+        res->orig_size = size;
+        res->intern.flags = 0;
+        erts_refc_init(&res->intern.refc, 1);
+    }
+
     return res;
 }
 
 ERTS_GLB_INLINE Binary *
 erts_bin_nrml_alloc(Uint size)
 {
-    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
-    Binary *res;
+    Binary *res = erts_bin_drv_alloc_fnf(size);
 
-    if (bsize < size) /* overflow */
-	erts_alloc_enomem(ERTS_ALC_T_BINARY, size);
-    res = erts_alloc(ERTS_ALC_T_BINARY, bsize);
-    ERTS_CHK_BIN_ALIGNMENT(res);
-    res->orig_size = size;
-    res->intern.flags = 0;
-    erts_refc_init(&res->intern.refc, 1);
-    return res;
+    if (res) {
+        return res;
+    }
+
+    erts_alloc_enomem(ERTS_ALC_T_BINARY, size);
 }
 
 ERTS_GLB_INLINE Binary *
 erts_bin_realloc_fnf(Binary *bp, Uint size)
 {
+    ErtsAlcType_t type;
     Binary *nbp;
-    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
-    ErtsAlcType_t type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
-	                                            : ERTS_ALC_T_BINARY;
+    Uint bsize;
+    
+    type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
+                                             : ERTS_ALC_T_BINARY;
     ASSERT((bp->intern.flags & BIN_FLAG_MAGIC) == 0);
-    if (bsize < size) /* overflow */
-	return NULL;
+
+    BINARY_OVERFLOW_CHECK(size);
+    bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
+
     nbp = erts_realloc_fnf(type, (void *) bp, bsize);
     ERTS_CHK_BIN_ALIGNMENT(nbp);
-    if (nbp)
-	nbp->orig_size = size;
+
+    if (nbp) {
+        nbp->orig_size = size;
+    }
+
     return nbp;
 }
 
 ERTS_GLB_INLINE Binary *
 erts_bin_realloc(Binary *bp, Uint size)
 {
-    Binary *nbp;
-    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
-    ErtsAlcType_t type = (bp->intern.flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
-	                                            : ERTS_ALC_T_BINARY;
-    ASSERT((bp->intern.flags & BIN_FLAG_MAGIC) == 0);
-    if (bsize < size) /* overflow */
-	erts_realloc_enomem(type, bp, size);
-    nbp = erts_realloc_fnf(type, (void *) bp, bsize);
-    if (!nbp)
-	erts_realloc_enomem(type, bp, bsize);
-    ERTS_CHK_BIN_ALIGNMENT(nbp);
-    nbp->orig_size = size;
-    return nbp;
+    Binary *nbp = erts_bin_realloc_fnf(bp, size);
+
+    if (nbp) {
+        return nbp;
+    }
+
+    if (bp->intern.flags & BIN_FLAG_DRV) {
+        erts_realloc_enomem(ERTS_ALC_T_DRV_BINARY, bp, size);
+    } else {
+        erts_realloc_enomem(ERTS_ALC_T_BINARY, bp, size);
+    }
 }
 
 ERTS_GLB_INLINE void