aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
authorVitaliy Batichko <[email protected]>2010-12-10 16:38:13 +0200
committerVitaliy Batichko <[email protected]>2010-12-10 17:39:56 +0200
commit384b85a75212cf73253f901a8c88d4286a5962ba (patch)
tree9649a4368f09f4f42d2ff2dac217da6d615676ef /lib
parent7992d6a6bfa0f460f8663c2d72d07b60755a4857 (diff)
downloadotp-384b85a75212cf73253f901a8c88d4286a5962ba.tar.gz
otp-384b85a75212cf73253f901a8c88d4286a5962ba.tar.bz2
otp-384b85a75212cf73253f901a8c88d4286a5962ba.zip
Fix wrong erl_compare_ext result comparing lists
Function erl_compare_ext of erl_interface library returns 1 instead -1 when comparing lists like [0] and [0, 1000]. Credits to: Evgeny Khirin <evgeny.khirin <at> erlios.com>
Diffstat (limited to 'lib')
-rw-r--r--lib/erl_interface/src/legacy/erl_marshal.c13
-rw-r--r--lib/erl_interface/test/erl_ext_SUITE_data/ext_test.c12
2 files changed, 20 insertions, 5 deletions
diff --git a/lib/erl_interface/src/legacy/erl_marshal.c b/lib/erl_interface/src/legacy/erl_marshal.c
index a6c2f64dd0..70949a7adf 100644
--- a/lib/erl_interface/src/legacy/erl_marshal.c
+++ b/lib/erl_interface/src/legacy/erl_marshal.c
@@ -1646,11 +1646,14 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
min = (i < j) ? i : j;
k = 0;
while (1) {
- if (k++ == min)
- return compare_top_ext(e1 , e2);
- if ((ret = compare_top_ext(e1 , e2)) == 0)
- continue;
- return ret;
+ if (k++ == min){
+ if (i == j) return 0;
+ if (i < j) return -1;
+ return 1;
+ }
+ if ((ret = compare_top_ext(e1 , e2)) == 0)
+ continue;
+ return ret;
}
case ERL_STRING_EXT:
i = (**e1 << 8) | ((*e1)[1]);
diff --git a/lib/erl_interface/test/erl_ext_SUITE_data/ext_test.c b/lib/erl_interface/test/erl_ext_SUITE_data/ext_test.c
index ba1a6c66da..59e0e0cce7 100644
--- a/lib/erl_interface/test/erl_ext_SUITE_data/ext_test.c
+++ b/lib/erl_interface/test/erl_ext_SUITE_data/ext_test.c
@@ -82,6 +82,11 @@ TESTCASE(compare_list) {
// erlang:term_to_binary([34,{a,n},a,erlang])
unsigned char term2[] = {131,108,0,0,0,4,97,34,104,2,100,0,1,97,100,0,1,110,100,0,1,97,100,0,6,101,114,108,97,110,103,106};
+ // erlang:term_to_binary([0])
+ unsigned char term3[] = {131,107,0,1,0};
+ // erlang:term_to_binary([0, 1000])
+ unsigned char term4[] = {131,108,0,0,0,2,97,0,98,0,0,3,232,106};
+
erl_init(NULL, 0);
start_a = term1;
start_b = term2;
@@ -90,6 +95,13 @@ TESTCASE(compare_list) {
test_compare_ext("lists", start_a, end_a, start_b, end_b, 1);
+ start_a = term3;
+ start_b = term4;
+ end_a = term3 + sizeof(term3);
+ end_b = term4 + sizeof(term4);
+
+ test_compare_ext("lists1", start_a, end_a, start_b, end_b, -1);
+
report(1);
}