aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
authorErlang/OTP <[email protected]>2019-07-02 13:44:00 +0200
committerErlang/OTP <[email protected]>2019-07-02 13:44:00 +0200
commita555c4c80c38b3e4240a2fcc54d913d0229193cd (patch)
treee7c7cbf6ccd8b7e58fd6dc490ae2539e5779c344 /erts/emulator/beam
parent4319a7f8bc93388ec540e954eb9bb73ea1eec4ef (diff)
parentc14e0eea80911e36fe45839af96bb9593c63bbb6 (diff)
downloadotp-a555c4c80c38b3e4240a2fcc54d913d0229193cd.tar.gz
otp-a555c4c80c38b3e4240a2fcc54d913d0229193cd.tar.bz2
otp-a555c4c80c38b3e4240a2fcc54d913d0229193cd.zip
Merge branch 'john/erts/lists_subtract_fixes/OTP-15938/OTP-15939' into maint-22
* john/erts/lists_subtract_fixes/OTP-15938/OTP-15939: erts: Fix integer overflow in loader erts: Fix integer overflow in list subtraction
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/beam_load.c10
-rw-r--r--erts/emulator/beam/erl_bif_lists.c15
2 files changed, 23 insertions, 2 deletions
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 941c3ebbbe..35f2ea6688 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4612,7 +4612,15 @@ typedef struct SortGenOpArg {
static int
genopargtermcompare(SortGenOpArg* a, SortGenOpArg* b)
{
- return CMP_TERM(a->term, b->term);
+ Sint res = CMP_TERM(a->term, b->term);
+
+ if (res < 0) {
+ return -1;
+ } else if (res > 0) {
+ return 1;
+ }
+
+ return 0;
}
static int
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index b23fa77f5f..fa2edfef1e 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -413,12 +413,25 @@ typedef struct {
#define ERTS_RBT_GET_LEFT(T) ((T)->left)
#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L))
#define ERTS_RBT_GET_KEY(T) ((T)->key)
-#define ERTS_RBT_CMP_KEYS(KX, KY) CMP_TERM(KX, KY)
+#define ERTS_RBT_CMP_KEYS(KX, KY) subtract_term_cmp((KX), (KY))
#define ERTS_RBT_WANT_LOOKUP_INSERT
#define ERTS_RBT_WANT_LOOKUP
#define ERTS_RBT_WANT_DELETE
#define ERTS_RBT_UNDEF
+/* erl_rbtree expects comparisons to return an int */
+static int subtract_term_cmp(Eterm a, Eterm b) {
+ Sint res = CMP_TERM(a, b);
+
+ if (res < 0) {
+ return -1;
+ } else if (res > 0) {
+ return 1;
+ }
+
+ return 0;
+}
+
#include "erl_rbtree.h"
static int subtract_continue(Process *p, ErtsSubtractContext *context);