Merge branch 'mikpe/avoid-oversize-bignums' into maint

* mikpe/avoid-oversize-bignums:
  don't leave a heap hole in erts_bs_get_integer_2
  don't create oversize bignums in binary matching

OTP-12556

4 files changed, 10 insertions, 2 deletions

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index cfc6146b0a..41c1b5d2c2 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4971,7 +4971,8 @@ get_tag_and_value(LoaderState* stp, Uint len_code,
 
     arity = count/sizeof(Eterm);
     *result = new_literal(stp, &hp, arity+1);
-    (void) bytes_to_big(bigbuf, count, neg, hp);
+    if (is_nil(bytes_to_big(bigbuf, count, neg, hp)))
+	goto load_error;
 
     if (bigbuf != default_buf) {
 	erts_free(ERTS_ALC_T_LOADER_TMP, (void *) bigbuf);
diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index de7d370938..d1e46e3063 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1900,6 +1900,8 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r)
 	*rwp = d;
 	rwp++;
     }
+    if (rsz > BIG_ARITY_MAX)
+	return NIL;
     if (xsgn) {
       *r = make_neg_bignum_header(rsz);
     }
diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c
index 73765772c8..53c21c40e1 100644
--- a/erts/emulator/beam/erl_bits.c
+++ b/erts/emulator/beam/erl_bits.c
@@ -403,7 +403,10 @@ erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuff
 	words_needed = 1+WSIZE(bytes);
 	hp = HeapOnlyAlloc(p, words_needed);
 	res = bytes_to_big(LSB, bytes, sgn, hp); 
-	if (is_small(res)) {
+	if (is_nil(res)) {
+	    p->htop = hp;
+	    res = THE_NON_VALUE;
+	} else if (is_small(res)) {
 	    p->htop = hp;
 	} else if ((actual = bignum_header_arity(*hp)+1) < words_needed) {
 	    p->htop = hp + actual;
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 9b9b4b2a62..45d1f7514e 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3056,6 +3056,8 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap,
 		    big = make_small(0);
 		} else {
 		    big = bytes_to_big(first, n, neg, hp);
+		    if (is_nil(big))
+			goto error;
 		    if (is_big(big)) {
 			hp += big_arity(big) + 1;
 		    }