term_to_binary use all 32 bits of INTEGER_EXT

Earlier, external format INTEGER_EXT was only produced for 28-bit
signed integers. Now full 32-bit signed integers are produced as
INTEGER_EXT to avoid the more costly SMALL_BIG_EXT format. Both old
and new code can read 32-bit INTEGER_EXT.

Also fixed integer encoding bugs in erl_interface erl_encode/erl_decode.
(Thanks to Alexander Demidenko for reporting)

2 files changed, 39 insertions, 22 deletions

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index f41b61d73d..466165f26f 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -65,11 +65,9 @@
 #  endif
 #endif
 
-/*
- * For backward compatibility reasons, only encode integers that
- * fit in 28 bits (signed) using INTEGER_EXT.
+/* Does Sint fit in Sint32?
  */
-#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2)
+#define IS_SSMALL32(x) (((Uint) (((x) >> (32-1)) + 1)) < 2)
 
 /*
  *   Valid creations for nodes are 1, 2, or 3. 0 can also be sent
@@ -1571,13 +1569,15 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 
 	case SMALL_DEF:
 	    {
+		/* From R14B we no longer restrict INTEGER_EXT to 28 bits,
+		 * as done earlier for backward compatibility reasons. */
 		Sint val = signed_val(obj);
 
 		if ((Uint)val < 256) {
 		    *ep++ = SMALL_INTEGER_EXT;
 		    put_int8(val, ep);
 		    ep++;
-		} else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) {
+		} else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) {
 		    *ep++ = INTEGER_EXT;
 		    put_int32(val, ep);
 		    ep += 4;
@@ -1599,18 +1599,32 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags)
 	    break;
 
 	case BIG_DEF:
-	    if ((n = big_bytes(obj)) < 256) {
-		*ep++ = SMALL_BIG_EXT;
-		put_int8(n, ep);
-		ep += 1;
-	    }
-	    else {
-		*ep++ = LARGE_BIG_EXT;
-		put_int32(n, ep);
-		ep += 4;
+	    {
+		int sign = big_sign(obj);
+		n = big_bytes(obj);
+		if (sizeof(Sint)==4 && n<=4) {
+		    Uint dig = big_digit(obj,0);		   
+		    Sint val = sign ? -dig : dig;
+		    if ((val<0) == sign) {
+			*ep++ = INTEGER_EXT;
+			put_int32(val, ep);
+			ep += 4;
+			break;
+		    }
+		}
+		if (n < 256) {
+		    *ep++ = SMALL_BIG_EXT;
+		    put_int8(n, ep);
+		    ep += 1;
+		}
+		else {
+		    *ep++ = LARGE_BIG_EXT;
+		    put_int32(n, ep);
+		    ep += 4;
+		}
+		*ep++ = sign;
+		ep = big_to_bytes(obj, ep);
 	    }
-	    *ep++ = big_sign(obj);
-	    ep = big_to_bytes(obj, ep);
 	    break;
 
 	case PID_DEF:
@@ -2687,7 +2701,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 
 		if ((Uint)val < 256)
 		    result += 1 + 1;		/* SMALL_INTEGER_EXT */
-		else if (sizeof(Sint) == 4 || IS_SSMALL28(val))
+		else if (sizeof(Sint) == 4 || IS_SSMALL32(val))
 		    result += 1 + 4;		/* INTEGER_EXT */
 		else {
 		    DeclareTmpHeapNoproc(tmp_big,2);
@@ -2699,7 +2713,10 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags)
 	    }
 	    break;
 	case BIG_DEF:
-	    if ((i = big_bytes(obj)) < 256)
+	    i = big_bytes(obj);
+	    if (sizeof(Sint)==4 && i <= 4 && (big_digit(obj,0)-big_sign(obj)) < (1<<31))
+		result += 1 + 4;          /* INTEGER_EXT */
+	    else if (i < 256)
 		result += 1 + 1 + 1 + i;  /* tag,size,sign,digits */
 	    else
 		result += 1 + 4 + 1 + i;  /* tag,size,sign,digits */
diff --git a/erts/emulator/test/hash_SUITE.erl b/erts/emulator/test/hash_SUITE.erl
index 85bdb8bff8..f5d1871bfb 100644
--- a/erts/emulator/test/hash_SUITE.erl
+++ b/erts/emulator/test/hash_SUITE.erl
@@ -480,14 +480,14 @@ otp_5292_test() ->
     S2 = md5([md5(hash_int(S, E, PH)) || {Start, N, Sz} <- d(), 
                                          {S, E} <- int(Start, N, Sz)]),
     ?line Comment = case S1 of 
-                        <<43,186,76,102,87,4,110,245,203,177,206,6,130,69,43,99>> ->
+			<<4,248,208,156,200,131,7,1,173,13,239,173,112,81,16,174>> ->
 			    ?line big = erlang:system_info(endian),
                             "Big endian machine";
-                        <<21,206,139,15,149,28,167,81,98,225,132,254,49,125,174,195>> ->
+                        <<180,28,33,231,239,184,71,125,76,47,227,241,78,184,176,233>> ->
 			    ?line little = erlang:system_info(endian),
                             "Little endian machine"
                     end,
-    ?line <<140,37,79,80,26,242,130,22,20,229,123,240,223,244,43,99>> = S2,
+    ?line <<124,81,198,121,174,233,19,137,10,83,33,80,226,111,238,99>> = S2,
     ?line 2 = erlang:hash(1, (1 bsl 27) -1),
     ?line {'EXIT', _} = (catch erlang:hash(1, (1 bsl 27))),
     {comment, Comment}.
@@ -507,7 +507,7 @@ hash_int(Start, End, F) ->
     {Start, End, md5(HL)}.
 
 md5(T) ->
-    erlang:md5(term_to_binary(T)).
+    erlang:md5(term_to_binary(T)).   
 
 bit_level_binaries() ->
     ?line [3511317,7022633,14044578,28087749,56173436,112344123,90467083|_] =