Merge branch 'sverk/ets-compress-bug' into maint

* sverk/ets-compress-bug:
  erts: Fail binary_to_term if bignum arity is too large
  erts: Fix bignum-bug in ETS with compressed option

OTP-9932

3 files changed, 34 insertions, 3 deletions

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 976f05c990..25ac790d81 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -1844,6 +1844,7 @@ dsize_t big_bytes(Eterm x)
 /*
 ** Load a bignum from bytes
 ** xsz is the number of bytes in xp
+** *r is untouched if number fits in small
 */
 Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r)
 {
@@ -1852,7 +1853,7 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r)
     ErtsDigit d;
     int i;
 
-    while(xsz >= sizeof(ErtsDigit)) {
+    while(xsz > sizeof(ErtsDigit)) {
 	d = 0;
 	for(i = sizeof(ErtsDigit); --i >= 0;)
 	    d = (d << 8) | xp[i];
@@ -1867,11 +1868,20 @@ Eterm bytes_to_big(byte *xp, dsize_t xsz, int xsgn, Eterm *r)
 	d = 0;
 	for(i = xsz; --i >= 0;)
 	    d = (d << 8) | xp[i];
+	if (++rsz == 1 && IS_USMALL(xsgn,d)) {
+	    if (xsgn) d = -d;
+	    return make_small(d);
+	}
 	*rwp = d;
 	rwp++;
-	rsz++;
     }
-    return big_norm(r, rsz, (short) xsgn);
+    if (xsgn) {
+      *r = make_neg_bignum_header(rsz);
+    }
+    else {
+      *r = make_pos_bignum_header(rsz);
+    }
+    return make_big(r);
 }
 
 /*
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index 152dbcf085..9d52ed4e98 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3118,6 +3118,9 @@ decoded_size(byte *ep, byte* endp, int internal_tags)
 	case LARGE_BIG_EXT:
 	    CHKSIZE(4);
 	    n = get_int32(ep);
+	    if (n > BIG_ARITY_MAX*sizeof(ErtsDigit)) {
+		return -1;
+	    }
 	    SKIP2(n,4+1);		/* skip, size,sign,digits */
 	    heap_size += 1+1+(n+sizeof(Eterm)-1)/sizeof(Eterm); /* XXX: 1 too much? */
 	    break;
diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl
index 101828fdef..59532b65a0 100644
--- a/lib/stdlib/test/ets_SUITE.erl
+++ b/lib/stdlib/test/ets_SUITE.erl
@@ -72,6 +72,7 @@
 	 exit_many_many_tables_owner/1]).
 -export([write_concurrency/1, heir/1, give_away/1, setopts/1]).
 -export([bad_table/1, types/1]).
+-export([otp_9932/1]).
 -export([otp_9423/1]).
 
 -export([init_per_testcase/2, end_per_testcase/2]).
@@ -145,6 +146,7 @@ all() ->
      exit_many_large_table_owner, exit_many_tables_owner,
      exit_many_many_tables_owner, write_concurrency, heir,
      give_away, setopts, bad_table, types,
+     otp_9932,
      otp_9423].
 
 groups() -> 
@@ -5434,6 +5436,22 @@ types_do(Opts) ->
     ?line verify_etsmem(EtsMem).
 
 
+%% OTP-9932: Memory overwrite when inserting large integers in compressed bag.
+%% Will crash with segv on 64-bit opt if not fixed.
+otp_9932(Config) when is_list(Config) ->
+    T = ets:new(xxx, [bag, compressed]),    
+    Fun = fun(N) -> 
+		  Key = {1316110174588445 bsl N,1316110174588583 bsl N},
+		  S = {Key, Key},
+		  true = ets:insert(T, S),
+		  [S] = ets:lookup(T, Key),
+		  true = ets:insert(T, S),
+		  [S] = ets:lookup(T, Key)
+	  end,
+    lists:foreach(Fun, lists:seq(0, 16)),
+    ets:delete(T).
+    
+
 otp_9423(doc) -> ["vm-deadlock caused by race between ets:delete and others on write_concurrency table"];
 otp_9423(Config) when is_list(Config) ->
     InitF = fun(_) -> {0,0} end,