Merge branch 'lukas/erts/fix_neg_of_int64_min/OTP-12097' into maint

* lukas/erts/fix_neg_of_int64_min/OTP-12097:
  erts: Fix neg int overflow when sint is min size

4 files changed, 25 insertions, 17 deletions

diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c
index 4d087bf967..e62caa6b22 100644
--- a/erts/emulator/beam/big.c
+++ b/erts/emulator/beam/big.c
@@ -274,6 +274,9 @@
 	_b = _b << _s;							\
 	_vn1 = _b >> H_EXP;						\
 	_vn0 = _b & LO_MASK;						\
+        /* Sometimes _s is 0 which triggers undefined behaviour for the \
+           (_a0>>(D_EXP-_s)) shift, but this is ok because the          \
+           & -s will make it all to 0 later anyways. */                 \
 	_un32 = (_a1 << _s) | ((_a0>>(D_EXP-_s)) & (-_s >> (D_EXP-1)));	\
 	_un10 = _a0 << _s;						\
 	_un1 = _un10 >> H_EXP;						\
@@ -1542,21 +1545,24 @@ Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp)
 Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 {
     Eterm *hp = *hpp;
+    Uint64 ux;
     int neg;
-    if (x >= 0)
+    if (x >= 0) {
 	neg = 0;
+        ux = x;
+    }
     else {
 	neg = 1;
-	x = -x;
+	ux = -(Uint64)x;
     }
 #if defined(ARCH_32) || HALFWORD_HEAP
-    if (x >= (((Uint64) 1) << 32)) {
+    if (ux >= (((Uint64) 1) << 32)) {
 	if (neg)
 	    *hp = make_neg_bignum_header(2);
 	else
 	    *hp = make_pos_bignum_header(2);
-	BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff));
-	BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff));
+	BIG_DIGIT(hp, 0) = (Uint) (ux & ((Uint) 0xffffffff));
+	BIG_DIGIT(hp, 1) = (Uint) ((ux >> 32) & ((Uint) 0xffffffff));
 	*hpp += 3;
     }
     else
@@ -1566,7 +1572,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp)
 	    *hp = make_neg_bignum_header(1);
 	else
 	    *hp = make_pos_bignum_header(1);
-	BIG_DIGIT(hp, 0) = (Uint) x;
+	BIG_DIGIT(hp, 0) = (Uint) ux;
 	*hpp += 2;
     }
     return make_big(hp);
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index d80111822e..da31876d75 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -101,7 +101,7 @@ typedef Uint  dsize_t;	 /* Vector size type */
 #define ERTS_SINT64_HEAP_SIZE(X)				\
   (IS_SSMALL((X))						\
    ? 0								\
-   : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(X)))
+   : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(Uint64)(X)))
 #define ERTS_UINT64_HEAP_SIZE(X)				\
   (IS_USMALL(0, (X)) ? 0 : ERTS_UINT64_BIG_HEAP_SIZE__((X)))
 
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index 7e0e825a0d..3bf78adce7 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1324,9 +1324,9 @@ static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp)
 		goto badarg;
 	    }
 	    if (len < 0) {
-		Sint lentmp = -len;
+		Uint lentmp = -(Uint)len;
 		/* overflow */
-		if (lentmp == len || lentmp < 0 || -lentmp != len) {
+		if ((Sint)lentmp < 0) {
 		    goto badarg;
 		}
 		len = lentmp;
@@ -1555,9 +1555,9 @@ BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen)
 	goto badarg;
     }
     if (len < 0) {
-	Sint lentmp = -len;
+	Uint lentmp = -(Uint)len;
 	/* overflow */
-	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	if ((Sint)lentmp < 0) {
 	    goto badarg;
 	}
 	len = lentmp;
@@ -1644,9 +1644,9 @@ BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is
 	goto badarg;
     }
     if (len < 0) {
-	Sint lentmp = -len;
+	Uint lentmp = -(Uint)len;
 	/* overflow */
-	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	if ((Sint)lentmp < 0) {
 	    goto badarg;
 	}
 	len = lentmp;
@@ -2213,9 +2213,9 @@ static BIF_RETTYPE binary_bin_to_list_common(Process *p,
 	goto badarg;
     }
     if (len < 0) {
-	Sint lentmp = -len;
+	Uint lentmp = -(Uint)len;
 	/* overflow */
-	if (lentmp == len || lentmp < 0 || -lentmp != len) {
+	if ((Sint)lentmp < 0) {
 	    goto badarg;
 	}
 	len = lentmp;
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 5e6d812242..88c4006934 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -2066,8 +2066,10 @@ erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2)
     system_cleanup(flush_async);
 
     save_statistics();
-
-    an = abs(n);
+    if (n < 0)
+        an = -(unsigned int)n;
+    else
+        an = n;
 
     if (erts_mtrace_enabled)
 	erts_mtrace_exit((Uint32) an);