From a055766e0ca9d2b4a5007f00b007b087e06bc7a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20H=C3=B6gberg?= Date: Wed, 26 Jun 2019 09:26:41 +0200 Subject: erts: Fix integer overflow in list subtraction CMP_TERM returned an `Sint`, which overflowed the `int` used in erl_rbtree for storing the comparison, causing list subtraction to behave strangely. --- erts/emulator/beam/erl_bif_lists.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'erts/emulator/beam/erl_bif_lists.c') diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index aaf262780f..b69949f9cc 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -244,12 +244,25 @@ typedef struct { #define ERTS_RBT_GET_LEFT(T) ((T)->left) #define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L)) #define ERTS_RBT_GET_KEY(T) ((T)->key) -#define ERTS_RBT_CMP_KEYS(KX, KY) CMP_TERM(KX, KY) +#define ERTS_RBT_CMP_KEYS(KX, KY) subtract_term_cmp((KX), (KY)) #define ERTS_RBT_WANT_LOOKUP_INSERT #define ERTS_RBT_WANT_LOOKUP #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_UNDEF +/* erl_rbtree expects comparisons to return an int */ +static int subtract_term_cmp(Eterm a, Eterm b) { + Sint res = CMP_TERM(a, b); + + if (res < 0) { + return -1; + } else if (res > 0) { + return 1; + } + + return 0; +} + #include "erl_rbtree.h" static int subtract_continue(Process *p, ErtsSubtractContext *context); -- cgit v1.2.3