Check for overflow when appending binaries, and error out with system_limit

This fixes the following bug:

A = <<0:((1 bsl 32)-8)>>,
B = <<2, 3>>.
B =:= <<A/binary,1,B/binary>>. %% Evaluated to true...

2 files changed, 55 insertions, 4 deletions

diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 01734c55d7..c9e09501ff 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -1324,7 +1324,14 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 	    goto badarg;
 	}
     }
+
+    if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) {
+        c_p->freason = SYSTEM_LIMIT;
+        return THE_NON_VALUE;
+    }
+
     used_size_in_bits = erts_bin_offset + build_size_in_bits;
+
     sb->is_writable = 0;	/* Make sure that no one else can write. */
     pb->size = NBYTES(used_size_in_bits);
     pb->flags |= PB_ACTIVE_WRITER;
@@ -1398,9 +1405,21 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term,
 		goto badarg;
 	    }
 	}
-	used_size_in_bits = erts_bin_offset + build_size_in_bits;
-	used_size_in_bytes = NBYTES(used_size_in_bits);
-	bin_size = 2*used_size_in_bytes;
+
+        if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) {
+            c_p->freason = SYSTEM_LIMIT;
+            return THE_NON_VALUE;
+        }
+
+        used_size_in_bits = erts_bin_offset + build_size_in_bits;
+        used_size_in_bytes = NBYTES(used_size_in_bits);
+
+        if(used_size_in_bits < (ERTS_UINT_MAX / 2)) {
+            bin_size = 2 * used_size_in_bytes;
+        } else {
+            bin_size = NBYTES(ERTS_UINT_MAX);
+        }
+
 	bin_size = (bin_size < 256) ? 256 : bin_size;
 
 	/*
@@ -1491,6 +1510,12 @@ erts_bs_private_append(Process* p, Eterm bin, Eterm build_size_term, Uint unit)
      * Calculate new size in bytes.
      */
     erts_bin_offset = 8*sb->size + sb->bitsize;
+
+    if((ERTS_UINT_MAX - build_size_in_bits) < erts_bin_offset) {
+        p->freason = SYSTEM_LIMIT;
+        return THE_NON_VALUE;
+    }
+
     pos_in_bits_after_build = erts_bin_offset + build_size_in_bits;
     pb->size = (pos_in_bits_after_build+7) >> 3;
     pb->flags |= PB_ACTIVE_WRITER;
diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl
index cadb30e1a4..06f3bc2ff8 100644
--- a/erts/emulator/test/bs_construct_SUITE.erl
+++ b/erts/emulator/test/bs_construct_SUITE.erl
@@ -911,5 +911,31 @@ append_unit_8(Bin) ->
 append_unit_16(Bin) ->
     <<Bin/binary-unit:16,0:1>>.
 
-    
+bs_add_overflow(Config) ->
+    case erlang:system_info(wordsize) of
+        8 ->
+            {skip, "64-bit architecture"};
+        4 ->
+            {'EXIT', {system_limit, _}} = (catch bs_add_overflow_signed()),
+            {'EXIT', {system_limit, _}} = (catch bs_add_overflow_unsigned()),
+            ok
+    end.
+
+bs_add_overflow_signed() ->
+    %% Produce a large result of bs_add that, if cast to signed int, would
+    %% overflow into a negative number that fits a smallnum.
+    Large = <<0:((1 bsl 30)-1)>>,
+    <<Large/bits, Large/bits, Large/bits, Large/bits,
+      Large/bits, Large/bits, Large/bits, Large/bits,
+      Large/bits>>.
+
+bs_add_overflow_unsigned() ->
+    %% Produce a large result of bs_add that goes beyond the limit of an
+    %% unsigned word. This used to succeed but produced an incorrect result
+    %% where B =:= C!
+    A = <<0:((1 bsl 32)-8)>>,
+    B = <<2, 3>>,
+    C = <<A/binary,1,B/binary>>,
+    true = byte_size(B) < byte_size(C).
+
 id(I) -> I.