Merge branch 'bjorn/erts/improve-beam-ops'

* bjorn/erts/improve-beam-ops:
  Optimize dispatch of loop_rec from recv_set
  Add missing -no_next directives
  Eliminate three arguments for erts_hibernate()
  Eliminate three arguments for the apply() helper
  Assign machine registers for X86-64
  Annotate arithmetic instructions with likely/unlikely
  Add annotations for likely/unlikely
  ops.tab: Mark infrequently used instructions as %cold
  Introduce '%warm' and beam_warm.h

13 files changed, 154 insertions, 86 deletions

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index bc7eb72221..cdc14a0c8b 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -545,6 +545,7 @@ OPCODE_TABLES += hipe/hipe_ops.tab hipe/hipe_instrs.tab
 endif
 
 $(TTF_DIR)/beam_cold.h \
+$(TTF_DIR)/beam_warm.h \
 $(TTF_DIR)/beam_hot.h \
 $(TTF_DIR)/beam_opcodes.c \
 $(TTF_DIR)/beam_opcodes.h \
diff --git a/erts/emulator/beam/arith_instrs.tab b/erts/emulator/beam/arith_instrs.tab
index 67cd7c6a2a..7c9cd47e28 100644
--- a/erts/emulator/beam/arith_instrs.tab
+++ b/erts/emulator/beam/arith_instrs.tab
@@ -28,7 +28,7 @@ OUTLINED_ARITH_2(Fail, Live, Name, BIF, Op1, Op2, Dst) {
     result = erts_gc_$Name (c_p, reg, live);
     HEAVY_SWAPIN;
     ERTS_HOLE_CHECK(c_p);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
@@ -49,10 +49,10 @@ plus.fetch(Op1, Op2) {
 }
 
 plus.execute(Fail, Live, Dst) {
-    if (is_both_small(PlusOp1, PlusOp2)) {
+    if (ERTS_LIKELY(is_both_small(PlusOp1, PlusOp2))) {
         Sint i = signed_val(PlusOp1) + signed_val(PlusOp2);
         ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-        if (MY_IS_SSMALL(i)) {
+        if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
             $Dst = make_small(i);
             $NEXT0();
         }
@@ -72,10 +72,10 @@ minus.fetch(Op1, Op2) {
 }
 
 minus.execute(Fail, Live, Dst) {
-    if (is_both_small(MinusOp1, MinusOp2)) {
+    if (ERTS_LIKELY(is_both_small(MinusOp1, MinusOp2))) {
         Sint i = signed_val(MinusOp1) - signed_val(MinusOp2);
         ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-        if (MY_IS_SSMALL(i)) {
+        if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
             $Dst = make_small(i);
             $NEXT0();
         }
@@ -98,10 +98,10 @@ increment.fetch(Src) {
 
 increment.execute(IncrementVal, Live, Dst) {
     increment_val = $IncrementVal;
-    if (is_small(increment_reg_val)) {
+    if (ERTS_LIKELY(is_small(increment_reg_val))) {
         Sint i = signed_val(increment_reg_val) + increment_val;
         ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
-        if (MY_IS_SSMALL(i)) {
+        if (ERTS_LIKELY(MY_IS_SSMALL(i))) {
             $Dst = make_small(i);
             $NEXT0();
         }
@@ -113,7 +113,7 @@ increment.execute(IncrementVal, Live, Dst) {
     result = erts_gc_mixed_plus(c_p, reg, live);
     HEAVY_SWAPIN;
     ERTS_HOLE_CHECK(c_p);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
@@ -137,12 +137,12 @@ i_m_div(Fail, Live, Op1, Op2, Dst) {
 i_int_div(Fail, Live, Op1, Op2, Dst) {
     Eterm op1 = $Op1;
     Eterm op2 = $Op2;
-    if (op2 == SMALL_ZERO) {
+    if (ERTS_UNLIKELY(op2 == SMALL_ZERO)) {
         c_p->freason = BADARITH;
         $BIF_ERROR_ARITY_2($Fail, BIF_intdiv_2, op1, op2);
-    } else if (is_both_small(op1, op2)) {
+    } else if (ERTS_LIKELY(is_both_small(op1, op2))) {
         Sint ires = signed_val(op1) / signed_val(op2);
-        if (MY_IS_SSMALL(ires)) {
+        if (ERTS_LIKELY(MY_IS_SSMALL(ires))) {
             $Dst = make_small(ires);
             $NEXT0();
         }
@@ -162,15 +162,15 @@ rem.fetch(Src1, Src2) {
 }
 
 rem.execute(Fail, Live, Dst) {
-     if (RemOp2 == SMALL_ZERO) {
-         c_p->freason = BADARITH;
-         $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2);
-     } else if (is_both_small(RemOp1, RemOp2)) {
-         $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2));
-         $NEXT0();
-     } else {
-         $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst);
-     }
+    if (ERTS_UNLIKELY(RemOp2 == SMALL_ZERO)) {
+        c_p->freason = BADARITH;
+        $BIF_ERROR_ARITY_2($Fail, BIF_rem_2, RemOp1, RemOp2);
+    } else if (ERTS_LIKELY(is_both_small(RemOp1, RemOp2))) {
+        $Dst = make_small(signed_val(RemOp1) % signed_val(RemOp2));
+        $NEXT0();
+    } else {
+        $OUTLINED_ARITH_2($Fail, $Live, int_rem, BIF_rem_2, RemOp1, RemOp2, $Dst);
+    }
 }
 
 i_band := band.fetch.execute;
@@ -185,7 +185,7 @@ band.fetch(Src1, Src2) {
 }
 
 band.execute(Fail, Live, Dst) {
-    if (is_both_small(BandOp1, BandOp2)) {
+    if (ERTS_LIKELY(is_both_small(BandOp1, BandOp2))) {
         /*
          * No need to untag -- TAG & TAG == TAG.
          */
@@ -196,7 +196,7 @@ band.execute(Fail, Live, Dst) {
 }
 
 i_bor(Fail, Live, Src1, Src2, Dst) {
-    if (is_both_small($Src1, $Src2)) {
+    if (ERTS_LIKELY(is_both_small($Src1, $Src2))) {
         /*
          * No need to untag -- TAG | TAG == TAG.
          */
@@ -207,7 +207,7 @@ i_bor(Fail, Live, Src1, Src2, Dst) {
 }
 
 i_bxor(Fail, Live, Src1, Src2, Dst) {
-    if (is_both_small($Src1, $Src2)) {
+    if (ERTS_LIKELY(is_both_small($Src1, $Src2))) {
         /*
          * TAG ^ TAG == 0.
          *
@@ -232,7 +232,7 @@ shift.setup_bsr(Src1, Src2) {
     Op1 = $Src1;
     Op2 = $Src2;
     shift_left_count = 0;
-    if (is_small(Op2)) {
+    if (ERTS_LIKELY(is_small(Op2))) {
         shift_left_count = -signed_val(Op2);
     } else if (is_big(Op2)) {
         /*
@@ -248,7 +248,7 @@ shift.setup_bsl(Src1, Src2) {
     Op1 = $Src1;
     Op2 = $Src2;
     shift_left_count = 0;
-    if (is_small(Op2)) {
+    if (ERTS_LIKELY(is_small(Op2))) {
         shift_left_count = signed_val(Op2);
     } else if (is_big(Op2)) {
         if (bignum_header_is_neg(*big_val(Op2))) {
@@ -271,10 +271,10 @@ shift.setup_bsl(Src1, Src2) {
 shift.execute(Fail, Live, Dst) {
     Uint big_words_needed;
 
-    if (is_small(Op1)) {
+    if (ERTS_LIKELY(is_small(Op1))) {
         Sint int_res = signed_val(Op1);
-        if (shift_left_count == 0 || int_res == 0) {
-            if (is_not_integer(Op2)) {
+        if (ERTS_UNLIKELY(shift_left_count == 0 || int_res == 0)) {
+            if (ERTS_UNLIKELY(is_not_integer(Op2))) {
                 goto shift_error;
             }
             if (int_res == 0) {
@@ -343,7 +343,7 @@ shift.execute(Fail, Live, Dst) {
                 HTOP += bignum_header_arity(*HTOP) + 1;
             }
             HEAP_SPACE_VERIFIED(0);
-            if (is_nil(Op1)) {
+            if (ERTS_UNLIKELY(is_nil(Op1))) {
                 /*
                  * This result must have been only slighty larger
                  * than allowed since it wasn't caught by the
@@ -381,7 +381,7 @@ shift.execute(Fail, Live, Dst) {
 
 i_int_bnot(Fail, Src, Live, Dst) {
     Eterm bnot_val = $Src;
-    if (is_small(bnot_val)) {
+    if (ERTS_LIKELY(is_small(bnot_val))) {
         bnot_val = make_small(~signed_val(bnot_val));
     } else {
         Uint live = $Live;
@@ -390,7 +390,7 @@ i_int_bnot(Fail, Src, Live, Dst) {
         bnot_val = erts_gc_bnot(c_p, reg, live);
         HEAVY_SWAPIN;
         ERTS_HOLE_CHECK(c_p);
-        if (is_nil(bnot_val)) {
+        if (ERTS_UNLIKELY(is_nil(bnot_val))) {
             $BIF_ERROR_ARITY_1($Fail, BIF_bnot_1, reg[live]);
         }
         $REFRESH_GEN_DEST();
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 2c37dc42b3..7ef0772703 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -392,9 +392,8 @@ static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa,
 				     Eterm* reg, Eterm func) NOINLINE;
 static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity,
 			      BeamInstr *I, Uint offs) NOINLINE;
-static BeamInstr* apply(Process* p, Eterm module, Eterm function,
-			Eterm args, Eterm* reg,
-			BeamInstr *I, Uint offs) NOINLINE;
+static BeamInstr* apply(Process* p, Eterm* reg,
+                        BeamInstr *I, Uint offs) NOINLINE;
 static BeamInstr* call_fun(Process* p, int arity,
 			   Eterm* reg, Eterm args) NOINLINE;
 static BeamInstr* apply_fun(Process* p, Eterm fun,
@@ -439,6 +438,12 @@ init_emulator(void)
 #  define REG_stop asm("%l3")
 #  define REG_I asm("%l4")
 #  define REG_fcalls asm("%l5")
+#elif defined(__GNUC__) && defined(__amd64__) && !defined(DEBUG)
+#  define REG_xregs asm("%r12")
+#  define REG_htop
+#  define REG_stop asm("%r13")
+#  define REG_I asm("%rbx")
+#  define REG_fcalls asm("%r14")
 #else
 #  define REG_xregs
 #  define REG_htop
@@ -870,6 +875,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
      goto do_schedule1;
  }
 
+#include "beam_warm.h"
+
  OpCase(normal_exit): {
      SWAPOUT;
      c_p->freason = EXC_NORMAL;
@@ -2174,13 +2181,14 @@ apply_bif_error_adjustment(Process *p, Export *ep,
 }
 
 static BeamInstr*
-apply(
-Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg,
-BeamInstr *I, Uint stack_offset)
+apply(Process* p, Eterm* reg, BeamInstr *I, Uint stack_offset)
 {
     int arity;
     Export* ep;
     Eterm tmp;
+    Eterm module = reg[0];
+    Eterm function = reg[1];
+    Eterm args = reg[2];
 
     /*
      * Check the arguments which should be of the form apply(Module,
@@ -2297,8 +2305,9 @@ fixed_apply(Process* p, Eterm* reg, Uint arity,
     if (is_not_atom(module)) goto error;
 
     /* Handle apply of apply/3... */
-    if (module == am_erlang && function == am_apply && arity == 3)
-	return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset);
+    if (module == am_erlang && function == am_apply && arity == 3) {
+	return apply(p, reg, I, stack_offset);
+    }
     
     /*
      * Get the index into the export table, or failing that the export
@@ -2319,10 +2328,13 @@ fixed_apply(Process* p, Eterm* reg, Uint arity,
 }
 
 int
-erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
+erts_hibernate(Process* c_p, Eterm* reg)
 {
     int arity;
     Eterm tmp;
+    Eterm module = reg[0];
+    Eterm function = reg[1];
+    Eterm args = reg[2];
 
     if (is_not_atom(module) || is_not_atom(function)) {
 	/*
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index cbbc2e88a1..ded6c6f1a4 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1250,7 +1250,11 @@ BIF_RETTYPE hibernate_3(BIF_ALIST_3)
      */
     Eterm reg[3];
 
-    if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) {
+    reg[0] = BIF_ARG_1;
+    reg[1] = BIF_ARG_2;
+    reg[2] = BIF_ARG_3;
+
+    if (erts_hibernate(BIF_P, reg)) {
         /*
          * If hibernate succeeded, TRAP. The process will be wait in a
          * hibernated state if its state is inactive (!ERTS_PSFLG_ACTIVE);
diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
index 5aa0523e06..3c95113907 100644
--- a/erts/emulator/beam/bif_instrs.tab
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -45,7 +45,7 @@ CALL_GUARD_BIF(BF, TmpReg, Dst) {
     ERTS_HOLE_CHECK(c_p);
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $Dst = result;
         $NEXT0();
     }
@@ -145,12 +145,12 @@ i_gc_bif1(Fail, Bif, Src, Live, Dst) {
     ERTS_HOLE_CHECK(c_p);
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
     }
-    if ($Fail != 0) {          /* Handle error in guard. */
+    if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
         $NEXT($Fail);
     }
 
@@ -195,13 +195,13 @@ i_gc_bif2(Fail, Bif, Live, Src1, Src2, Dst) {
     ERTS_HOLE_CHECK(c_p);
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
     }
 
-    if ($Fail != 0) {          /* Handle error in guard. */
+    if (ERTS_LIKELY($Fail != 0)) { /* Handle error in guard. */
         $NEXT($Fail);
     }
 
@@ -249,14 +249,14 @@ i_gc_bif3(Fail, Bif, Live, Src2, Src3, Dst) {
     ERTS_HOLE_CHECK(c_p);
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         $REFRESH_GEN_DEST();
         $Dst = result;
         $NEXT0();
     }
 
     /* Handle error in guard. */
-    if ($Fail != 0) {
+    if (ERTS_LIKELY($Fail != 0)) {
         $NEXT($Fail);
     }
 
@@ -325,7 +325,7 @@ call_bif(Exp) {
         ERTS_MSACC_UPDATE_CACHE_X();
     }
     ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         r(0) = result;
         CHECK_TERM(r(0));
         $NEXT0();
@@ -370,7 +370,7 @@ send() {
     PROCESS_MAIN_CHK_LOCKS(c_p);
     HTOP = HEAP_TOP(c_p);
     FCALLS = c_p->fcalls;
-    if (is_value(result)) {
+    if (ERTS_LIKELY(is_value(result))) {
         r(0) = result;
         CHECK_TERM(r(0));
     } else if (c_p->freason == TRAP) {
@@ -520,7 +520,7 @@ nif_bif.epilogue() {
     SWAPIN;  /* There might have been a garbage collection. */
     FCALLS = c_p->fcalls;
     ERTS_DBG_CHK_REDS(c_p, FCALLS);
-    if (is_value(nif_bif_result)) {
+    if (ERTS_LIKELY(is_value(nif_bif_result))) {
         r(0) = nif_bif_result;
         CHECK_TERM(r(0));
         SET_I(c_p->cp);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 27a6202bb7..c72ae32f2c 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1386,7 +1386,7 @@ Uint erts_current_reductions(Process* current, Process *p);
 
 int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p);
 
-int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);
+int erts_hibernate(Process* c_p, Eterm* reg);
 
 ERTS_GLB_FORCE_INLINE int erts_is_literal(Eterm tptr, Eterm *ptr);
 
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab
index 1af01e53bd..9c3e615cea 100644
--- a/erts/emulator/beam/instrs.tab
+++ b/erts/emulator/beam/instrs.tab
@@ -159,7 +159,7 @@ i_move_call_ext_last(Dest, StackOffset, Src) {
 APPLY(I, Deallocate) {
     //| -no_next
     HEAVY_SWAPOUT;
-    next = apply(c_p, r(0), x(1), x(2), reg, $I, $Deallocate);
+    next = apply(c_p, reg, $I, $Deallocate);
     HEAVY_SWAPIN;
 }
 
@@ -171,7 +171,7 @@ HANDLE_APPLY_ERROR() {
 i_apply() {
     BeamInstr *next;
     $APPLY(NULL, 0);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call(next);
     }
     $HANDLE_APPLY_ERROR();
@@ -180,7 +180,7 @@ i_apply() {
 i_apply_last(Deallocate) {
     BeamInstr *next;
     $APPLY(I, $Deallocate);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call_last(next, $Deallocate);
     }
     $HANDLE_APPLY_ERROR();
@@ -189,7 +189,7 @@ i_apply_last(Deallocate) {
 i_apply_only() {
     BeamInstr *next;
     $APPLY(I, 0);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call_only(next);
     }
     $HANDLE_APPLY_ERROR();
@@ -205,7 +205,7 @@ FIXED_APPLY(Arity, I, Deallocate) {
 apply(Arity) {
     BeamInstr *next;
     $FIXED_APPLY($Arity, NULL, 0);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call(next);
     }
     $HANDLE_APPLY_ERROR();
@@ -214,7 +214,7 @@ apply(Arity) {
 apply_last(Arity, Deallocate) {
     BeamInstr *next;
     $FIXED_APPLY($Arity, I, $Deallocate);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $i_call_last(next, $Deallocate);
     }
     $HANDLE_APPLY_ERROR();
@@ -238,7 +238,7 @@ DISPATCH_FUN(I) {
 i_apply_fun() {
     BeamInstr *next;
     $APPLY_FUN();
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         SET_CP(c_p, $NEXT_INSTRUCTION);
         $DISPATCH_FUN(next);
     }
@@ -248,7 +248,7 @@ i_apply_fun() {
 i_apply_fun_last(Deallocate) {
     BeamInstr *next;
     $APPLY_FUN();
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $deallocate($Deallocate);
         $DISPATCH_FUN(next);
     }
@@ -258,7 +258,7 @@ i_apply_fun_last(Deallocate) {
 i_apply_fun_only() {
     BeamInstr *next;
     $APPLY_FUN();
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $DISPATCH_FUN(next);
     }
     $HANDLE_APPLY_FUN_ERROR();
@@ -274,7 +274,7 @@ CALL_FUN(Fun) {
 i_call_fun(Fun) {
     BeamInstr *next;
     $CALL_FUN($Fun);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         SET_CP(c_p, $NEXT_INSTRUCTION);
         $DISPATCH_FUN(next);
     }
@@ -284,7 +284,7 @@ i_call_fun(Fun) {
 i_call_fun_last(Fun, Deallocate) {
     BeamInstr *next;
     $CALL_FUN($Fun);
-    if (next != NULL) {
+    if (ERTS_LIKELY(next != NULL)) {
         $deallocate($Deallocate);
         $DISPATCH_FUN(next);
     }
@@ -378,7 +378,7 @@ element_group.fetch(Src) {
 
 element_group.execute(Fail, Index, Dst) {
     element_index = $Index;
-    if (is_small(element_index) && is_tuple(element_tuple)) {
+    if (ERTS_LIKELY(is_small(element_index) && is_tuple(element_tuple))) {
         Eterm* tp = tuple_val(element_tuple);
 
         if ((signed_val(element_index) >= 1) &&
@@ -402,7 +402,7 @@ fast_element_group.fetch(Src) {
 }
 
 fast_element_group.execute(Fail, Index, Dst) {
-    if (is_tuple(fast_element_tuple)) {
+    if (ERTS_LIKELY(is_tuple(fast_element_tuple))) {
         Eterm* tp = tuple_val(fast_element_tuple);
         Eterm pos = $Index;	/* Untagged integer >= 1 */
         if (pos <= arityval(*tp)) {
@@ -906,5 +906,6 @@ i_raise() {
         c_p->freason = PRIMARY_EXCEPTION(s->freason);
     }
     goto find_func_info;
+    //| -no_next
 }
 
diff --git a/erts/emulator/beam/macros.tab b/erts/emulator/beam/macros.tab
index 41dc761e90..bac96be7d3 100644
--- a/erts/emulator/beam/macros.tab
+++ b/erts/emulator/beam/macros.tab
@@ -42,7 +42,7 @@ JUMP(Fail) {
 
 GC_TEST(Ns, Nh, Live) {
     Uint need = $Nh + $Ns;
-    if (E - HTOP < need) {
+    if (ERTS_UNLIKELY(E - HTOP < need)) {
        SWAPOUT;
        PROCESS_MAIN_CHK_LOCKS(c_p);
        FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, $Live, FCALLS);
@@ -55,7 +55,7 @@ GC_TEST(Ns, Nh, Live) {
 
 GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) {
     Uint need = $NeedHeap;
-    if (E - HTOP < need) {
+    if (ERTS_UNLIKELY(E - HTOP < need)) {
        SWAPOUT;
        reg[$Live] = $PreserveTerm;
        PROCESS_MAIN_CHK_LOCKS(c_p);
@@ -99,7 +99,13 @@ FAIL_BODY() {
 
 FAIL_HEAD_OR_BODY(Fail) {
     //| -no_prefetch
-    if ($Fail) {
+
+    /*
+     * In a correctly working program, we expect failures in
+     * guards to be more likely than failures in bodies.
+     */
+
+    if (ERTS_LIKELY($Fail)) {
         $FAIL($Fail);
     }
     goto find_func_info;
@@ -122,7 +128,7 @@ SYSTEM_LIMIT(Fail) {
 
 BIF_ERROR_ARITY_1(Fail, BIF, Op1) {
     //| -no_prefetch
-    if ($Fail) {
+    if (ERTS_LIKELY($Fail)) {
         $FAIL($Fail);
     }
     reg[0] = $Op1;
@@ -133,7 +139,7 @@ BIF_ERROR_ARITY_1(Fail, BIF, Op1) {
 
 BIF_ERROR_ARITY_2(Fail, BIF, Op1, Op2) {
     //| -no_prefetch
-    if ($Fail) {
+    if (ERTS_LIKELY($Fail)) {
         $FAIL($Fail);
     }
     reg[0] = $Op1;
diff --git a/erts/emulator/beam/msg_instrs.tab b/erts/emulator/beam/msg_instrs.tab
index 509143268b..093d48c64c 100644
--- a/erts/emulator/beam/msg_instrs.tab
+++ b/erts/emulator/beam/msg_instrs.tab
@@ -66,7 +66,9 @@ i_recv_set() {
     if (c_p->msg.mark == (BeamInstr *) ($NEXT_INSTRUCTION)) {
         c_p->msg.save = c_p->msg.saved_last;
     }
-    /* Fall through to the loop_rec/2 instruction */
+    SET_I($NEXT_INSTRUCTION);
+    goto loop_rec_top__;
+    //| -no_next
 }
 
 i_loop_rec(Dest) {
@@ -79,6 +81,10 @@ i_loop_rec(Dest) {
 
     ErtsMessage* msgp;
 
+    /* Entry point from recv_set */
+ loop_rec_top__:
+    ;
+
     /*
      * We need to disable GC while matching messages
      * in the queue. This since messages with data outside
@@ -87,7 +93,8 @@ i_loop_rec(Dest) {
     ASSERT(!(c_p->flags & F_DELAY_GC));
     c_p->flags |= F_DELAY_GC;
 
-loop_rec__:
+    /* Entry point from loop_rec_end */
+ loop_rec__:
 
     PROCESS_MAIN_CHK_LOCKS(c_p);
 
@@ -285,6 +292,7 @@ timeout() {
 TIMEOUT_VALUE() {
     c_p->freason = EXC_TIMEOUT_VALUE;
     goto find_func_info;
+    //| -no_next
 }
 
 i_wait_error_locked() {
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index b6e995fdbe..d848c1cceb 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -59,6 +59,7 @@ put_tuple u==0 d => too_old_compiler
 # All the other instructions.
 #
 
+%cold
 label L
 i_func_info I a a I
 int_code_end
@@ -68,6 +69,7 @@ i_debug_breakpoint
 i_return_time_trace
 i_return_to_trace
 i_yield
+%hot
 
 return
 
@@ -217,6 +219,11 @@ is_number Fail Literal=q => move Literal x | is_number Fail x
 
 jump f
 
+#
+# Expection rasing instructions. Infrequently executed.
+#
+
+%cold
 case_end NotInX=cy => move NotInX x | case_end x
 badmatch NotInX=cy => move NotInX x | badmatch x
 
@@ -238,6 +245,12 @@ i_raise
 badarg j
 system_limit j
 
+%hot
+
+#
+# Move instructions.
+#
+
 move C=cxy x==0 | jump Lbl => move_jump Lbl C
 
 move_jump f ncxy
@@ -404,8 +417,10 @@ wait_timeout_unlocked s f
 wait_timeout_locked_int I f
 wait_timeout_locked s f
 
+%cold
 i_wait_error
 i_wait_error_locked
+%hot
 
 send
 
@@ -515,6 +530,7 @@ put_list s s d
 # Some more only used by the emulator
 #
 
+%cold
 normal_exit
 continue_exit
 apply_bif
@@ -522,6 +538,7 @@ call_nif
 call_error_handler
 error_action_code
 return_trace
+%hot
 
 #
 # Instruction transformations & folded instructions.
@@ -939,9 +956,11 @@ i_apply_fun
 i_apply_fun_last P
 i_apply_fun_only
 
+%cold
 i_hibernate
 
 i_perf_counter
+%hot
 
 call_bif e
 
@@ -1040,7 +1059,7 @@ func_info M F A => i_func_info u M F A
 # New bit syntax matching (R11B).
 # ================================================================
 
-%cold
+%warm
 bs_start_match2 Fail=f ica X Y D => jump Fail
 bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D
 i_bs_start_match2 xy f I I x
@@ -1138,7 +1157,7 @@ i_bs_validate_unicode_retract j s s
 #
 # Constructing binaries
 #
-%cold
+%warm
 
 bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail
 
diff --git a/erts/emulator/beam/trace_instrs.tab b/erts/emulator/beam/trace_instrs.tab
index dfd1d16d58..c71f2ef003 100644
--- a/erts/emulator/beam/trace_instrs.tab
+++ b/erts/emulator/beam/trace_instrs.tab
@@ -102,7 +102,7 @@ i_yield() {
 
 i_hibernate() {
     HEAVY_SWAPOUT;
-    if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+    if (erts_hibernate(c_p, reg)) {
         FCALLS = c_p->fcalls;
         c_p->flags &= ~F_HIBERNATE_SCHED;
         goto do_schedule;
diff --git a/erts/emulator/hipe/hipe_ops.tab b/erts/emulator/hipe/hipe_ops.tab
index 96e4c0da91..19a3820a6a 100644
--- a/erts/emulator/hipe/hipe_ops.tab
+++ b/erts/emulator/hipe/hipe_ops.tab
@@ -23,4 +23,7 @@ hipe_trap_call_closure
 hipe_trap_return
 hipe_trap_throw
 hipe_trap_resume
+
+%cold
 hipe_call_count
+%hot
diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops
index 6c54ab3421..bcc92472d4 100755
--- a/erts/emulator/utils/beam_makeops
+++ b/erts/emulator/utils/beam_makeops
@@ -20,13 +20,16 @@
 #
 use strict;
 use vars qw($BEAM_FORMAT_NUMBER);
+use constant COLD => 0;
+use constant WARM => 1;
+use constant HOT => 2;
 
 $BEAM_FORMAT_NUMBER = undef;
 
 my $target = \&emulator_output;
 my $outdir = ".";		# Directory for output files.
 my $verbose = 0;
-my $hot = 1;
+my $hotness = 1;
 my $num_file_opcodes = 0;
 my $wordsize = 32;
 my %defs;			# Defines (from command line).
@@ -344,13 +347,16 @@ while (<>) {
     }
 
     #
-    # Handle %hot/%cold.
+    # Handle %hot, %warm, and %cold.
     # 
     if (/^\%hot/) {
-	$hot = 1;
+	$hotness = HOT;
 	next;
+    } elsif (/^\%warm/) {
+	$hotness = WARM;
+        next;
     } elsif (/^\%cold/) {
-	$hot = 0;
+	$hotness = COLD;
 	next;
     }
 
@@ -438,7 +444,7 @@ while (<>) {
     if (defined $gen_opnum{$name,$arity} and $obsolete[$gen_opnum{$name,$arity}]) {
 	error("specific instructions may not be specified for obsolete instructions");
     }
-    save_specific_ops($name, $arity, $hot, @args);
+    save_specific_ops($name, $arity, $hotness, @args);
     if (defined $op_num) {
 	error("specific instructions must not be numbered");
     } elsif (!defined($gen_arity{$name}) && !defined($unnumbered{$name,$arity})) {
@@ -801,12 +807,17 @@ sub emulator_output {
     $name = "$outdir/beam_hot.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
     comment('C');
-    print_code(1);
+    print_code(HOT);
+
+    $name = "$outdir/beam_warm.h";
+    open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
+    comment('C');
+    print_code(WARM);
 
     $name = "$outdir/beam_cold.h";
     open(STDOUT, ">$name") || die "Failed to open $name for writing: $!\n";
     comment('C');
-    print_code(0);
+    print_code(COLD);
 }
 
 sub init_item {
@@ -1039,14 +1050,16 @@ sub combine_micro_instructions {
 
     # Now generate code for each group.
     foreach my $group (sort keys %groups) {
-        my($code,@labels) = combine_instruction_group($group, @{$groups{$group}});
-        push @generated_code, [1,$code,@labels];
+        my($hotness,$code,@labels) =
+            combine_instruction_group($group, @{$groups{$group}});
+        push @generated_code, [$hotness,$code,@labels];
     }
 }
 
 sub combine_instruction_group {
     my($group,@in_instrs) = @_;
     my $gcode = '';             # Code for the entire group.
+    my $group_hotness = COLD;
 
     # Get code for the head of the group (if any).
     my $head_name = "$group.head";
@@ -1075,7 +1088,8 @@ sub combine_instruction_group {
         error("no $specific_key instruction")
             unless defined $specific_op_ref;
         foreach my $specific_op (@$specific_op_ref) {
-            my($name, $hot, @args) = @{$specific_op};
+            my($name, $hotness, @args) = @{$specific_op};
+            $group_hotness = $hotness unless $group_hotness >= $hotness;
             my $offset = 0;
             my @rest = @args;
             my @new_subs;
@@ -1182,7 +1196,7 @@ sub combine_instruction_group {
         $offset = $order_to_offset{$slots[$i+1]} if $i < $#slots;
     }
 
-    ("{\n$gcode\n}\n\n",@opcase_labels);
+    ($group_hotness,"{\n$gcode\n}\n\n",@opcase_labels);
 }
 
 sub micro_label {