aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorErlang/OTP <[email protected]>2019-07-02 13:44:44 +0200
committerErlang/OTP <[email protected]>2019-07-02 13:44:44 +0200
commit7e5af024d07418ddc9c1e7068bf1aa2d99e1b178 (patch)
treec0822a2f55bc61e40f93991bbc674f1408e648cc
parent348e328375fb774b3fa919ffd1c4811367406516 (diff)
parentc14e0eea80911e36fe45839af96bb9593c63bbb6 (diff)
downloadotp-7e5af024d07418ddc9c1e7068bf1aa2d99e1b178.tar.gz
otp-7e5af024d07418ddc9c1e7068bf1aa2d99e1b178.tar.bz2
otp-7e5af024d07418ddc9c1e7068bf1aa2d99e1b178.zip
Merge branch 'john/erts/lists_subtract_fixes/OTP-15938/OTP-15939' into maint-21
* john/erts/lists_subtract_fixes/OTP-15938/OTP-15939: erts: Fix integer overflow in loader erts: Fix integer overflow in list subtraction
-rw-r--r--erts/emulator/beam/beam_load.c10
-rw-r--r--erts/emulator/beam/erl_bif_lists.c15
-rw-r--r--lib/stdlib/test/lists_SUITE.erl11
3 files changed, 34 insertions, 2 deletions
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index e61199a8fd..725b8006f7 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -4547,7 +4547,15 @@ typedef struct SortGenOpArg {
static int
genopargtermcompare(SortGenOpArg* a, SortGenOpArg* b)
{
- return CMP_TERM(a->term, b->term);
+ Sint res = CMP_TERM(a->term, b->term);
+
+ if (res < 0) {
+ return -1;
+ } else if (res > 0) {
+ return 1;
+ }
+
+ return 0;
}
static int
diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c
index aaf262780f..b69949f9cc 100644
--- a/erts/emulator/beam/erl_bif_lists.c
+++ b/erts/emulator/beam/erl_bif_lists.c
@@ -244,12 +244,25 @@ typedef struct {
#define ERTS_RBT_GET_LEFT(T) ((T)->left)
#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L))
#define ERTS_RBT_GET_KEY(T) ((T)->key)
-#define ERTS_RBT_CMP_KEYS(KX, KY) CMP_TERM(KX, KY)
+#define ERTS_RBT_CMP_KEYS(KX, KY) subtract_term_cmp((KX), (KY))
#define ERTS_RBT_WANT_LOOKUP_INSERT
#define ERTS_RBT_WANT_LOOKUP
#define ERTS_RBT_WANT_DELETE
#define ERTS_RBT_UNDEF
+/* erl_rbtree expects comparisons to return an int */
+static int subtract_term_cmp(Eterm a, Eterm b) {
+ Sint res = CMP_TERM(a, b);
+
+ if (res < 0) {
+ return -1;
+ } else if (res > 0) {
+ return 1;
+ }
+
+ return 0;
+}
+
#include "erl_rbtree.h"
static int subtract_continue(Process *p, ErtsSubtractContext *context);
diff --git a/lib/stdlib/test/lists_SUITE.erl b/lib/stdlib/test/lists_SUITE.erl
index 984b51e7ae..9a0fe4b5ca 100644
--- a/lib/stdlib/test/lists_SUITE.erl
+++ b/lib/stdlib/test/lists_SUITE.erl
@@ -2586,6 +2586,15 @@ subtract(Config) when is_list(Config) ->
[1,2,3,4,5,6,7,8,9,9999,10000,20,21,22] =
sub(lists:seq(1, 10000)++[20,21,22], lists:seq(10, 9998)),
+ %% ERL-986; an integer overflow relating to term comparison
+ %% caused subtraction to be inconsistent.
+ Ids = [2985095936,47540628,135460048,1266126295,240535295,
+ 115724671,161800351,4187206564,4178142725,234897063,
+ 14773162,6662515191,133150693,378034895,1874402262,
+ 3507611978,22850922,415521280,253360400,71683243],
+
+ [] = id(Ids) -- id(Ids),
+
%% Floats/integers.
[42.0,42.0] = sub([42.0,42,42.0], [42,42,42]),
[1,2,3,4,43.0] = sub([1,2,3,4,5,42.0,43.0], [42.0,5]),
@@ -2613,6 +2622,8 @@ subtract(Config) when is_list(Config) ->
ok.
+id(I) -> I.
+
sub_non_matching(A, B) ->
A = sub(A, B).