Avoid erts_cmp jump in atom, int and float comparisons

Given the function definition below:

    check(X) when X >= 0, X <= 20 -> true.

@nox has originally noticed that perfoming lt and ge
guard tests were performing slower than they should be.

Further investigation revealed that most of the cost
was in jumping to the erts_cmp function. This patch
brings the operations already inlined in erts_cmp
into the emulator, removing the jump cost.

After applying these changes, invoking the check/1
function defined above 30000 times with different
values from 0 to 20 has fallen from 367us to 213us
(measured as average of 3 runs). This is a
considerably improvement over Erlang 18 which takes
556us on average.

Floats have also dropped their time from 1126us
(on Erlang 18) to 613us.

4 files changed, 38 insertions, 24 deletions

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index fd2adac676..099c00bcf6 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -176,7 +176,7 @@ atom_alloc(Atom* tmpl)
 
     /*
      * Precompute ordinal value of first 3 bytes + 7 bits.
-     * This is used by utils.c:cmp_atoms().
+     * This is used by utils.c:erts_cmp_atoms().
      * We cannot use the full 32 bits of the first 4 bytes,
      * since we use the sign of the difference between two
      * ordinal values to represent their relative order.
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 4d7b00b032..659ff3ad2f 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -718,10 +718,10 @@ void** beam_ops;
 #define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
 #define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; }
 #define NotEqualExact(X, Y, Action) if (EQ(X,Y)) { Action; }
-#define Equal(X, Y, Action) if (!CMP_EQ(X,Y)) { Action; }
-#define NotEqual(X, Y, Action) if (!CMP_NE(X,Y)) { Action; }
-#define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; }
-#define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; }
+#define Equal(X, Y, Action) CMP_EQ_ACTION(X,Y,Action)
+#define NotEqual(X, Y, Action) CMP_NE_ACTION(X,Y,Action)
+#define IsLessThan(X, Y, Action) CMP_LT_ACTION(X,Y,Action)
+#define IsGreaterEqual(X, Y, Action) CMP_GE_ACTION(X,Y,Action)
 
 #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }
 
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 4058d63eaf..b8746f7edb 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -161,7 +161,9 @@ int eq(Eterm, Eterm);
 
 #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))
 
+int erts_cmp_atoms(Eterm a, Eterm b);
 Sint erts_cmp(Eterm, Eterm, int, int);
+Sint erts_cmp_compound(Eterm, Eterm, int, int);
 Sint cmp(Eterm a, Eterm b);
 #define CMP(A,B)                         erts_cmp(A,B,0,0)
 #define CMP_TERM(A,B)                    erts_cmp(A,B,1,0)
@@ -174,17 +176,29 @@ Sint cmp(Eterm a, Eterm b);
 #define cmp_ge(a,b)          (CMP((a),(b)) >= 0)
 #define cmp_gt(a,b)          (CMP((a),(b)) >  0)
 
-#define cmp_lt_term(a,b)     (CMP_TERM((a),(b)) <  0)
-#define cmp_le_term(a,b)     (CMP_TERM((a),(b)) <= 0)
-#define cmp_ge_term(a,b)     (CMP_TERM((a),(b)) >= 0)
-#define cmp_gt_term(a,b)     (CMP_TERM((a),(b)) >  0)
-
-#define CMP_LT(a,b)          ((a) != (b) && cmp_lt((a),(b)))
-#define CMP_GE(a,b)          ((a) == (b) || cmp_ge((a),(b)))
 #define CMP_EQ(a,b)          ((a) == (b) || cmp_eq((a),(b)))
-#define CMP_NE(a,b)          ((a) != (b) && cmp_ne((a),(b)))
 
-#define CMP_LT_TERM(a,b)     ((a) != (b) && cmp_lt_term((a),(b)))
-#define CMP_GE_TERM(a,b)     ((a) == (b) || cmp_ge_term((a),(b)))
+#define CMP_EQ_ACTION(X,Y,Action)	\
+    if ((X) != (Y)) { CMP_SPEC((X),(Y),!=,Action,1); }
+#define CMP_NE_ACTION(X,Y,Action)	\
+    if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),==,Action,1); }
+#define CMP_GE_ACTION(X,Y,Action)	\
+    if ((X) != (Y)) { CMP_SPEC((X),(Y),<,Action,0); }
+#define CMP_LT_ACTION(X,Y,Action)	\
+    if ((X) == (Y)) { Action; } else { CMP_SPEC((X),(Y),>=,Action,0); }
+
+#define CMP_SPEC(X,Y,Op,Action,EqOnly)				\
+    if (is_atom(X) && is_atom(Y)) {				\
+	if (erts_cmp_atoms(X, Y) Op 0) { Action; };		\
+    } else if (is_both_small(X, Y)) {				\
+	if (signed_val(X) Op signed_val(Y)) { Action; };	\
+    } else if (is_float(X) && is_float(Y)) {			\
+        FloatDef af, bf;					\
+        GET_DOUBLE(X, af);					\
+        GET_DOUBLE(Y, bf);					\
+        if (af.fd Op bf.fd) { Action; };			\
+    } else {							\
+	if (erts_cmp_compound(X,Y,0,EqOnly) Op 0) { Action; };	\
+    }
 
 #endif
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index ef851d840d..6aad1ff778 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -2991,7 +2991,7 @@ static int cmpbytes(byte *s1, int l1, byte *s2, int l2)
 
 #define float_comp(x,y)    (((x)<(y)) ? -1 : (((x)==(y)) ? 0 : 1))
 
-static int cmp_atoms(Eterm a, Eterm b)
+int erts_cmp_atoms(Eterm a, Eterm b)
 {
     Atom *aa = atom_tab(atom_val(a));
     Atom *bb = atom_tab(atom_val(b));
@@ -3010,12 +3010,12 @@ Sint cmp(Eterm a, Eterm b)
     return erts_cmp(a, b, 0, 0);
 }
 
-static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only);
+Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only);
 
 Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
 {
     if (is_atom(a) && is_atom(b)) {
-        return cmp_atoms(a, b);
+        return erts_cmp_atoms(a, b);
     } else if (is_both_small(a, b)) {
         return (signed_val(a) - signed_val(b));
     } else if (is_float(a) && is_float(b)) {
@@ -3032,7 +3032,7 @@ Sint erts_cmp(Eterm a, Eterm b, int exact, int eq_only)
  * exact = 1 -> term-based compare
  * exact = 0 -> arith-based compare
  */
-static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
+Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
 {
 #define PSTACK_TYPE struct erts_cmp_hashmap_state
     struct erts_cmp_hashmap_state {
@@ -3089,7 +3089,7 @@ static Sint erts_cmp_compound(Eterm a, Eterm b, int exact, int eq_only)
     do {								\
 	if((AN) != (BN)) {						\
             if((AN)->sysname != (BN)->sysname)				\
-                RETURN_NEQ(cmp_atoms((AN)->sysname, (BN)->sysname));	\
+                RETURN_NEQ(erts_cmp_atoms((AN)->sysname, (BN)->sysname));	\
 	    ASSERT((AN)->creation != (BN)->creation);			\
 	    RETURN_NEQ(((AN)->creation < (BN)->creation) ? -1 : 1);	\
 	}								\
@@ -3107,7 +3107,7 @@ tailrecur_ne:
     /* deal with majority (?) cases by brute-force */
     if (is_atom(a)) {
 	if (is_atom(b)) {
-	    ON_CMP_GOTO(cmp_atoms(a, b));
+	    ON_CMP_GOTO(erts_cmp_atoms(a, b));
 	}
     } else if (is_both_small(a, b)) {
 	ON_CMP_GOTO(signed_val(a) - signed_val(b));
@@ -3341,10 +3341,10 @@ tailrecur_ne:
 		    Export* a_exp = *((Export **) (export_val(a) + 1));
 		    Export* b_exp = *((Export **) (export_val(b) + 1));
 
-		    if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
+		    if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) {
 			RETURN_NEQ(j);
 		    }
-		    if ((j = cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) {
+		    if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) {
 			RETURN_NEQ(j);
 		    }
 		    ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]);
@@ -3659,7 +3659,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */
 	b = *bb++;
 	if (!is_same(a, b)) {
 	    if (is_atom(a) && is_atom(b)) {
-		if ((j = cmp_atoms(a, b)) != 0) {
+		if ((j = erts_cmp_atoms(a, b)) != 0) {
 		    goto not_equal;
 		}
 	    } else if (is_both_small(a, b)) {