erts: Fix integer overflow in list subtraction

CMP_TERM returned an `Sint`, which overflowed the `int` used in
erl_rbtree for storing the comparison, causing list subtraction to
behave strangely.

1 files changed, 14 insertions, 1 deletions

diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index aaf262780f..b69949f9cc 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -244,12 +244,25 @@ typedef struct {
 #define ERTS_RBT_GET_LEFT(T) ((T)->left)
 #define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L))
 #define ERTS_RBT_GET_KEY(T) ((T)->key)
-#define ERTS_RBT_CMP_KEYS(KX, KY) CMP_TERM(KX, KY)
+#define ERTS_RBT_CMP_KEYS(KX, KY) subtract_term_cmp((KX), (KY))
 #define ERTS_RBT_WANT_LOOKUP_INSERT
 #define ERTS_RBT_WANT_LOOKUP
 #define ERTS_RBT_WANT_DELETE
 #define ERTS_RBT_UNDEF
 
+/* erl_rbtree expects comparisons to return an int */
+static int subtract_term_cmp(Eterm a, Eterm b) {
+    Sint res = CMP_TERM(a, b);
+
+    if (res < 0) {
+        return -1;
+    } else if (res > 0) {
+        return 1;
+    }
+
+    return 0;
+}
+
 #include "erl_rbtree.h"
 
 static int subtract_continue(Process *p, ErtsSubtractContext *context);